From 625fdfd1a2ac3a7094d79fdae2d59081d4c49c51 Mon Sep 17 00:00:00 2001 From: TJ Date: Fri, 25 Oct 2024 16:29:17 +0900 Subject: [PATCH 1/4] first commit --- .gitignore | 13 +-- README.md | 44 ++++++++-- docs/{worklogs => dev-logs}/2024-10-21.md | 0 docs/dev-logs/2024-10-25.md | 9 ++ docs/dev/readme.md | 0 docs/dev/typesense.md | 14 ++++ lib/migration/typesense.ex | 42 ++++++++++ lib/migration/typesense/note.ex | 29 +++++++ lib/migration/typesense/photo.ex | 39 +++++++++ lib/save_it/bot.ex | 48 +++++++++++ lib/save_it/note_service.ex | 31 +++++++ lib/save_it/typesense_photo.ex | 10 +-- lib/small_sdk/typesense.ex | 34 +++++++- lib/small_sdk/typesense_admin.ex | 31 ------- .../2024-10-24_create_photos_collection.exs | 1 + .../2024-10-25_create_notes_collections.exs | 1 + priv/typesense/reset.exs | 3 +- zeabur/template.yaml | 83 +++++++++++++++++++ 18 files changed, 382 insertions(+), 50 deletions(-) rename docs/{worklogs => dev-logs}/2024-10-21.md (100%) create mode 100644 docs/dev-logs/2024-10-25.md create mode 100644 docs/dev/readme.md create mode 100644 docs/dev/typesense.md create mode 100644 lib/migration/typesense.ex create mode 100644 lib/migration/typesense/note.ex create mode 100644 lib/migration/typesense/photo.ex create mode 100644 lib/save_it/note_service.ex create mode 100644 priv/typesense/2024-10-24_create_photos_collection.exs create mode 100644 priv/typesense/2024-10-25_create_notes_collections.exs create mode 100644 zeabur/template.yaml diff --git a/.gitignore b/.gitignore index c81d2b0..1df64ff 100644 --- a/.gitignore +++ b/.gitignore @@ -25,10 +25,13 @@ save_it-*.tar # Temporary files, for example, from tests. /tmp/ .DS_Store -data -dev.sh -start.sh -run.sh -nohup.out +# data _local +data + +# scripts +_local* +_stag* +_dev* +nohup.out diff --git a/README.md b/README.md index fdf3744..72bd025 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ A telegram bot can Save photos and Search photos +## Features + - [x] Save photos via a link - [x] Search photos using semantic search - [x] Find similar photos by photo @@ -27,13 +29,8 @@ messages: ``` /search cat - /search dog - /search girl - -/similar photo - /similar photo ``` @@ -58,10 +55,45 @@ mix deps.get ``` ```sh -# run +# start typesense +docker compose up +``` + +```sh +# modify env export TELEGRAM_BOT_TOKEN= export TYPESENSE_URL= export TYPESENSE_API_KEY= iex -S mix run --no-halt ``` + +Pro Tips: create shell script for fast run app + +1. touch start.sh + +```sh +#!/bin/sh + +export TELEGRAM_BOT_TOKEN= + +export TYPESENSE_URL= +export TYPESENSE_API_KEY= + +export GOOGLE_OAUTH_CLIENT_ID= +export GOOGLE_OAUTH_CLIENT_SECRET= + +iex -S mix run --no-halt +``` + +2. execute permission + +```sh +chmod +x start.sh +``` + +3. run + +```sh +./start.sh +``` diff --git a/docs/worklogs/2024-10-21.md b/docs/dev-logs/2024-10-21.md similarity index 100% rename from docs/worklogs/2024-10-21.md rename to docs/dev-logs/2024-10-21.md diff --git a/docs/dev-logs/2024-10-25.md b/docs/dev-logs/2024-10-25.md new file mode 100644 index 0000000..ddc6e58 --- /dev/null +++ b/docs/dev-logs/2024-10-25.md @@ -0,0 +1,9 @@ +# 2024-10-25 + +## Req call typesense API alway :timeout, but typesense was updated. + +```elixir +** (MatchError) no match of right hand side value: {:error, %Req.TransportError{reason: :timeout}} + (save_it 0.2.0-rc.1) lib/migration/typesense.ex:11: Migration.Typesense.create_collection!/1 + priv/typesense/reset.exs:3: (file) +``` diff --git a/docs/dev/readme.md b/docs/dev/readme.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/dev/typesense.md b/docs/dev/typesense.md new file mode 100644 index 0000000..c564cf3 --- /dev/null +++ b/docs/dev/typesense.md @@ -0,0 +1,14 @@ +# Typesense + +## Typesense API Errors + +``` +# 400 Bad Request - The request could not be understood due to malformed syntax. +# 401 Unauthorized - Your API key is wrong. +# 404 Not Found - The requested resource is not found. +# 409 Conflict - When a resource already exists. +# 422 Unprocessable Entity - Request is well-formed, but cannot be processed. +# 503 Service Unavailable - We’re temporarily offline. Please try again later. +``` + +docs: https://typesense.org/docs/27.1/api/api-errors.html#api-errors diff --git a/lib/migration/typesense.ex b/lib/migration/typesense.ex new file mode 100644 index 0000000..13eb509 --- /dev/null +++ b/lib/migration/typesense.ex @@ -0,0 +1,42 @@ +defmodule Migration.Typesense do + def list_collections() do + req = build_request("/collections") + {:ok, res} = Req.get(req) + + res.body + end + + def create_collection!(schema) do + req = build_request("/collections") + {:ok, res} = Req.post(req, json: schema) + + res.body + end + + def delete_collection!(collection_name) do + req = build_request("/collections/#{collection_name}") + {:ok, res} = Req.delete(req) + + res.body + end + + defp get_env() do + url = Application.fetch_env!(:save_it, :typesense_url) + api_key = Application.fetch_env!(:save_it, :typesense_api_key) + + {url, api_key} + end + + defp build_request(path) do + {url, api_key} = get_env() + + Req.new( + base_url: url, + url: path, + headers: [ + {"Content-Type", "application/json"}, + {"X-TYPESENSE-API-KEY", api_key} + ] + ) + end +end diff --git a/lib/migration/typesense/note.ex b/lib/migration/typesense/note.ex new file mode 100644 index 0000000..22006c7 --- /dev/null +++ b/lib/migration/typesense/note.ex @@ -0,0 +1,29 @@ +defmodule Migration.Typesense.Note do + alias Migration.Typesense + + @notes_schema %{ + "name" => "notes", + "fields" => [ + # TODO: 第一步先实现文本，今后再考虑图片 + %{"name" => "content", "type" => "string"}, + # references photos.id + # note: 抉择：这个 app 核心是给予图片的视觉笔记，暂时不考虑单独 text 的笔记 + # %{"name" => "photo_id", "type" => "string"}, + # note: 既然不能实现 RDB reference，那么就直接存储 file_id + %{"name" => "file_id", "type" => "string"}, + %{"name" => "belongs_to_id", "type" => "string"}, + %{"name" => "inserted_at", "type" => "int64"}, + %{"name" => "updated_at", "type" => "int64"} + ], + "default_sorting_field" => "inserted_at" + } + + def create_collection!() do + Typesense.create_collection!(@notes_schema) + end + + def reset!() do + Typesense.delete_collection!(@notes_schema["name"]) + Typesense.create_collection!(@notes_schema) + end +end diff --git a/lib/migration/typesense/photo.ex b/lib/migration/typesense/photo.ex new file mode 100644 index 0000000..ae1abe5 --- /dev/null +++ b/lib/migration/typesense/photo.ex @@ -0,0 +1,39 @@ +defmodule Migration.Typesense.Photo do + alias Migration.Typesense + + @photos_schema %{ + "name" => "photos", + "fields" => [ + # image: base64 encoded string + %{"name" => "image", "type" => "image", "store" => false}, + %{ + "name" => "image_embedding", + "type" => "float[]", + "embed" => %{ + "from" => ["image"], + "model_config" => %{ + "model_name" => "ts/clip-vit-b-p32" + } + } + }, + %{"name" => "caption", "type" => "string", "optional" => true, "facet" => false}, + # "telegram:///" + # TODO: 不能再简单的 reset 了，reset 会导致数据丢失，应该合理 migrate 数据 + %{"name" => "url", "type" => "string"}, + # chat.id -> string + %{"name" => "belongs_to_id", "type" => "string"}, + # unix timestamp + %{"name" => "inserted_at", "type" => "int64"} + ], + "default_sorting_field" => "inserted_at" + } + + def create_collection!() do + Typesense.create_collection!(@photos_schema) + end + + def reset!() do + Typesense.delete_collection!(@photos_schema["name"]) + Typesense.create_collection!(@photos_schema) + end +end diff --git a/lib/save_it/bot.ex b/lib/save_it/bot.ex index 3390101..df73336 100644 --- a/lib/save_it/bot.ex +++ b/lib/save_it/bot.ex @@ -7,6 +7,8 @@ defmodule SaveIt.Bot do alias SaveIt.TypesensePhoto + alias SaveIt.NoteService + alias SmallSdk.Telegram @bot :save_it_bot @@ -24,6 +26,7 @@ defmodule SaveIt.Bot do command("start") command("search", description: "Search photos") + command("note", description: "Add a note to a photo") command("similar", description: "Find similar photos") command("about", description: "About the bot") @@ -115,6 +118,10 @@ defmodule SaveIt.Bot do end end + def handle({:command, :search, %{chat: chat, text: nil}}, _context) do + send_message(chat.id, "What do you want to search? animal, food, etc.") + end + def handle({:command, :search, %{chat: chat, text: text}}, _context) when is_binary(text) do q = String.trim(text) @@ -130,6 +137,47 @@ defmodule SaveIt.Bot do end end + # dev-notes: never reach here, text never be nil, "" + # def handle({:command, :note, %{chat: chat, text: nil}}, _context) do + # end + + def handle( + {:command, :note, %{chat: chat, text: text, reply_to_message: reply_to_message}} = msg, + _context + ) + when is_binary(text) do + Logger.debug("photo: #{inspect(reply_to_message.photo)}") + + file_id = reply_to_message.photo |> List.last() |> Map.get(:file_id) + + case file_id do + nil -> + send_message(chat.id, "Please reply to a photo to add a note.") + + _ -> + case String.trim(text) do + "" -> + send_message(chat.id, "What note do you want to add?") + + note_content -> + # photo_url = photo_url(chat.id, file_id) + # TypesensePhoto.update_photo!(photo_url, %{"note" => text}) + + note = + NoteService.create_note!(%{ + content: note_content, + file_id: file_id, + belongs_to_id: chat.id + }) + + case note do + nil -> send_message(chat.id, "Failed to add note.") + _ -> send_message(chat.id, "Note added successfully.") + end + end + end + end + def handle({:command, :similar, %{chat: chat, photo: nil}}, _context) do send_message(chat.id, "Upload a photo to find similar photos.") end diff --git a/lib/save_it/note_service.ex b/lib/save_it/note_service.ex new file mode 100644 index 0000000..6a742d7 --- /dev/null +++ b/lib/save_it/note_service.ex @@ -0,0 +1,31 @@ +defmodule SaveIt.NoteService do + require Logger + + alias SmallSdk.Typesense + + def create_note!(%{ + content: content, + file_id: file_id, + belongs_to_id: belongs_to_id + }) do + now_unix = DateTime.utc_now() |> DateTime.to_unix() + + note_create_input = + %{ + content: content, + file_id: file_id + } + |> Map.put(:belongs_to_id, Integer.to_string(belongs_to_id)) + |> Map.put(:inserted_at, now_unix) + |> Map.put(:updated_at, now_unix) + + doc = + Typesense.create_document!( + "notes", + note_create_input + ) + + Logger.debug("doc: #{inspect(doc)}") + doc + end +end diff --git a/lib/save_it/typesense_photo.ex b/lib/save_it/typesense_photo.ex index 587b84d..4a75db5 100644 --- a/lib/save_it/typesense_photo.ex +++ b/lib/save_it/typesense_photo.ex @@ -46,7 +46,8 @@ defmodule SaveIt.TypesensePhoto do req = build_request("/multi_search") {:ok, res} = Req.post(req, json: req_body) - res.body["results"] |> typesense_results_to_documents() + # FIXME: nil check + res.body["results"] |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) end def search_similar_photos!(photo_id, opts \\ []) when is_binary(photo_id) do @@ -69,11 +70,8 @@ defmodule SaveIt.TypesensePhoto do req = build_request("/multi_search") {:ok, res} = Req.post(req, json: req_body) - res.body["results"] |> typesense_results_to_documents() - end - - defp typesense_results_to_documents(results) do - results |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) + # FIXME: nil check + res.body["results"] |> hd() |> Map.get("hits") |> Enum.map(&Map.get(&1, "document")) end defp get_env() do diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index 6260713..24d400d 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -1,11 +1,43 @@ defmodule SmallSdk.Typesense do require Logger + def handle_response(res) do + case res do + %Req.Response{status: 200} -> + res.body + + %Req.Response{status: 201} -> + res.body + + %Req.Response{status: 400} -> + Logger.error("Bad Request: #{inspect(res.body)}") + raise "Bad Request" + + %Req.Response{status: 401} -> + raise "Unauthorized" + + %Req.Response{status: 404} -> + raise "Not Found" + + %Req.Response{status: 409} -> + raise "Conflict" + + %Req.Response{status: 422} -> + raise "Unprocessable Entity" + + %Req.Response{status: 503} -> + raise "Service Unavailable" + + _ -> + raise "Unknown error" + end + end + def create_document!(collection_name, document) do req = build_request("/collections/#{collection_name}/documents") {:ok, res} = Req.post(req, json: document) - res.body + handle_response(res) end def get_document(collection_name, document_id) do diff --git a/lib/small_sdk/typesense_admin.ex b/lib/small_sdk/typesense_admin.ex index 1b5fd6d..a123240 100644 --- a/lib/small_sdk/typesense_admin.ex +++ b/lib/small_sdk/typesense_admin.ex @@ -1,35 +1,4 @@ defmodule SmallSdk.TypesenseAdmin do - @photos_schema %{ - "name" => "photos", - "fields" => [ - # image: base64 encoded string - %{"name" => "image", "type" => "image", "store" => false}, - %{ - "name" => "image_embedding", - "type" => "float[]", - "embed" => %{ - "from" => ["image"], - "model_config" => %{ - "model_name" => "ts/clip-vit-b-p32" - } - } - }, - %{"name" => "caption", "type" => "string", "optional" => true, "facet" => false}, - # "telegram:///" - %{"name" => "url", "type" => "string"}, - # chat.id -> string - %{"name" => "belongs_to_id", "type" => "string"}, - # unix timestamp - %{"name" => "inserted_at", "type" => "int64"} - ], - "default_sorting_field" => "inserted_at" - } - - def reset() do - delete_collection!(@photos_schema["name"]) - create_collection!(@photos_schema) - end - def create_collection!(schema) do req = build_request("/collections") {:ok, res} = Req.post(req, json: schema) diff --git a/priv/typesense/2024-10-24_create_photos_collection.exs b/priv/typesense/2024-10-24_create_photos_collection.exs new file mode 100644 index 0000000..aa0aff4 --- /dev/null +++ b/priv/typesense/2024-10-24_create_photos_collection.exs @@ -0,0 +1 @@ +Migration.Typesense.Photo.create_collection!() diff --git a/priv/typesense/2024-10-25_create_notes_collections.exs b/priv/typesense/2024-10-25_create_notes_collections.exs new file mode 100644 index 0000000..4b028b0 --- /dev/null +++ b/priv/typesense/2024-10-25_create_notes_collections.exs @@ -0,0 +1 @@ +Migration.Typesense.Note.create_collection!() diff --git a/priv/typesense/reset.exs b/priv/typesense/reset.exs index 00773d2..885e5dc 100644 --- a/priv/typesense/reset.exs +++ b/priv/typesense/reset.exs @@ -1,3 +1,4 @@ # mix run priv/typesense/reset.exs -SmallSdk.TypesenseAdmin.reset() +Migration.Typesense.Photo.reset!() +Migration.Typesense.Note.reset!() diff --git a/zeabur/template.yaml b/zeabur/template.yaml new file mode 100644 index 0000000..d0c3d30 --- /dev/null +++ b/zeabur/template.yaml @@ -0,0 +1,83 @@ +# yaml-language-server: $schema=https://schema.zeabur.app/template.json +apiVersion: zeabur.com/v1 +kind: Template +metadata: + name: SaveIt +spec: + description: SaveIt is a telegram bot that helps you save photos. + icon: https://github.com/user-attachments/assets/fae196b8-716e-4be7-a8c2-3b141984c0e5 + tags: + - Bot + - Telegram + - Photos Storage + - Search Engine + coverImage: + + readme: |- + # SaveIt + A telegram bot can Save photos and Search photos + + ## Features + - Save photos via a link + - Search photos using semantic search + - Find similar photos by photo + + ## Learn more + https://github.com/ThaddeusJiang/save_it + + + + services: + - name: save_it + icon: https://github.com/user-attachments/assets/fae196b8-716e-4be7-a8c2-3b141984c0e5 + template: GIT + spec: + source: + source: GITHUB + repo: 831394769 + branch: main + rootDirectory: / + env: + TELEGRAM_BOT_TOKEN: + default: "" + expose: false + TYPESENSE_URL: + default: http://typesense.zeabur.internal:8108 + expose: false + TYPESENSE_API_KEY: + default: ${TYPESENSE_API_KEY} + expose: false + GOOGLE_OAUTH_CLIENT_ID: + default: "" + expose: false + GOOGLE_OAUTH_CLIENT_SECRET: + default: "" + expose: false + configs: [] + - name: typesense + icon: https://typesense.org/docs/images/typesense_logo.svg + template: PREBUILT_V2 + spec: + source: + image: typesense/typesense:27.1 + ports: + - id: web + port: 8108 + type: HTTP + volumes: + - id: data + dir: /data + instructions: + - type: PASSWORD + title: Typesense API Key + content: ${TYPESENSE_API_KEY} + category: Credentials + env: + TYPESENSE_API_KEY: + default: ${PASSWORD} + expose: true + TYPESENSE_DATA_DIR: + default: /data + expose: false + + configs: [] From 256dde8b953e40857003c72da8b09bb2f3892bca Mon Sep 17 00:00:00 2001 From: TJ Date: Fri, 25 Oct 2024 16:32:05 +0900 Subject: [PATCH 2/4] refactor --- lib/migration/typesense.ex | 14 ++++---- lib/save_it/bot.ex | 20 +++++------ .../{typesense_photo.ex => photo_service.ex} | 2 +- lib/small_sdk/typesense_admin.ex | 35 ------------------- 4 files changed, 18 insertions(+), 53 deletions(-) rename lib/save_it/{typesense_photo.ex => photo_service.ex} (98%) delete mode 100644 lib/small_sdk/typesense_admin.ex diff --git a/lib/migration/typesense.ex b/lib/migration/typesense.ex index 13eb509..8bea87e 100644 --- a/lib/migration/typesense.ex +++ b/lib/migration/typesense.ex @@ -1,11 +1,4 @@ defmodule Migration.Typesense do - def list_collections() do - req = build_request("/collections") - {:ok, res} = Req.get(req) - - res.body - end - def create_collection!(schema) do req = build_request("/collections") {:ok, res} = Req.post(req, json: schema) @@ -20,6 +13,13 @@ defmodule Migration.Typesense do res.body end + def list_collections() do + req = build_request("/collections") + {:ok, res} = Req.get(req) + + res.body + end + defp get_env() do url = Application.fetch_env!(:save_it, :typesense_url) api_key = Application.fetch_env!(:save_it, :typesense_api_key) diff --git a/lib/save_it/bot.ex b/lib/save_it/bot.ex index df73336..60120a5 100644 --- a/lib/save_it/bot.ex +++ b/lib/save_it/bot.ex @@ -5,7 +5,7 @@ defmodule SaveIt.Bot do alias SaveIt.GoogleDrive alias SaveIt.GoogleOAuth2DeviceFlow - alias SaveIt.TypesensePhoto + alias SaveIt.PhotoService alias SaveIt.NoteService @@ -131,7 +131,7 @@ defmodule SaveIt.Bot do send_message(chat.id, "What do you want to search? animal, food, etc.") _ -> - photos = TypesensePhoto.search_photos!(q, belongs_to_id: chat.id) + photos = PhotoService.search_photos!(q, belongs_to_id: chat.id) answer_photos(chat.id, photos) end @@ -142,7 +142,7 @@ defmodule SaveIt.Bot do # end def handle( - {:command, :note, %{chat: chat, text: text, reply_to_message: reply_to_message}} = msg, + {:command, :note, %{chat: chat, text: text, reply_to_message: reply_to_message}}, _context ) when is_binary(text) do @@ -161,7 +161,7 @@ defmodule SaveIt.Bot do note_content -> # photo_url = photo_url(chat.id, file_id) - # TypesensePhoto.update_photo!(photo_url, %{"note" => text}) + # PhotoService.update_photo!(photo_url, %{"note" => text}) note = NoteService.create_note!(%{ @@ -197,7 +197,7 @@ defmodule SaveIt.Bot do chat_id = chat.id typesense_photo = - TypesensePhoto.create_photo!(%{ + PhotoService.create_photo!(%{ image: Base.encode64(photo_file_content), caption: "", url: photo_url(bot_id, file.file_id), @@ -205,7 +205,7 @@ defmodule SaveIt.Bot do }) photos = - TypesensePhoto.search_similar_photos!( + PhotoService.search_similar_photos!( typesense_photo["id"], distance_threshold: 0.1, belongs_to_id: chat_id @@ -235,7 +235,7 @@ defmodule SaveIt.Bot do end typesense_photo = - TypesensePhoto.create_photo!(%{ + PhotoService.create_photo!(%{ image: Base.encode64(photo_file_content), caption: caption, url: photo_url(bot_id, file.file_id), @@ -245,7 +245,7 @@ defmodule SaveIt.Bot do case caption do "" -> photos = - TypesensePhoto.search_similar_photos!( + PhotoService.search_similar_photos!( typesense_photo["id"], distance_threshold: 0.4, belongs_to_id: chat_id @@ -255,7 +255,7 @@ defmodule SaveIt.Bot do _ -> photos = - TypesensePhoto.search_similar_photos!( + PhotoService.search_similar_photos!( typesense_photo["id"], distance_threshold: 0.1, belongs_to_id: chat_id @@ -483,7 +483,7 @@ defmodule SaveIt.Bot do {:file_content, file_content, _file_name} -> Base.encode64(file_content) end - TypesensePhoto.create_photo!(%{ + PhotoService.create_photo!(%{ image: image_base64, caption: file_name, url: photo_url(bot_id, file_id), diff --git a/lib/save_it/typesense_photo.ex b/lib/save_it/photo_service.ex similarity index 98% rename from lib/save_it/typesense_photo.ex rename to lib/save_it/photo_service.ex index 4a75db5..b7aa2d8 100644 --- a/lib/save_it/typesense_photo.ex +++ b/lib/save_it/photo_service.ex @@ -1,4 +1,4 @@ -defmodule SaveIt.TypesensePhoto do +defmodule SaveIt.PhotoService do require Logger alias SmallSdk.Typesense diff --git a/lib/small_sdk/typesense_admin.ex b/lib/small_sdk/typesense_admin.ex deleted file mode 100644 index a123240..0000000 --- a/lib/small_sdk/typesense_admin.ex +++ /dev/null @@ -1,35 +0,0 @@ -defmodule SmallSdk.TypesenseAdmin do - def create_collection!(schema) do - req = build_request("/collections") - {:ok, res} = Req.post(req, json: schema) - - res.body - end - - def delete_collection!(collection_name) do - req = build_request("/collections/#{collection_name}") - {:ok, res} = Req.delete(req) - - res.body - end - - defp get_env() do - url = Application.fetch_env!(:save_it, :typesense_url) - api_key = Application.fetch_env!(:save_it, :typesense_api_key) - - {url, api_key} - end - - defp build_request(path) do - {url, api_key} = get_env() - - Req.new( - base_url: url, - url: path, - headers: [ - {"Content-Type", "application/json"}, - {"X-TYPESENSE-API-KEY", api_key} - ] - ) - end -end From c9bacc8bc79fdcffa946a9477c3ed633229e7c7b Mon Sep 17 00:00:00 2001 From: TJ Date: Fri, 25 Oct 2024 18:42:29 +0900 Subject: [PATCH 3/4] create & update notes --- .gitignore | 3 +- config/runtime.exs | 2 - lib/migration/typesense/note.ex | 1 + lib/migration/typesense/photo.ex | 6 +-- lib/save_it/bot.ex | 68 ++++++++++++++------------------ lib/save_it/note_service.ex | 60 +++++++++++++++++++++------- lib/save_it/photo_service.ex | 2 +- lib/small_sdk/typesense.ex | 29 ++++++++++++-- 8 files changed, 105 insertions(+), 66 deletions(-) diff --git a/.gitignore b/.gitignore index 1df64ff..1c7fa49 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ data # scripts _local* -_stag* _dev* +_stag* +_prod* nohup.out diff --git a/config/runtime.exs b/config/runtime.exs index 047283f..faaef97 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -9,5 +9,3 @@ config :save_it, :typesense_api_key, System.get_env("TYPESENSE_API_KEY", "xyz") # optional config :save_it, :google_oauth_client_id, System.get_env("GOOGLE_OAUTH_CLIENT_ID") config :save_it, :google_oauth_client_secret, System.get_env("GOOGLE_OAUTH_CLIENT_SECRET") - -config :save_it, :web_url, System.get_env("WEB_URL", "http://localhost:4000") diff --git a/lib/migration/typesense/note.ex b/lib/migration/typesense/note.ex index 22006c7..727e73a 100644 --- a/lib/migration/typesense/note.ex +++ b/lib/migration/typesense/note.ex @@ -10,6 +10,7 @@ defmodule Migration.Typesense.Note do # note: 抉择：这个 app 核心是给予图片的视觉笔记，暂时不考虑单独 text 的笔记 # %{"name" => "photo_id", "type" => "string"}, # note: 既然不能实现 RDB reference，那么就直接存储 file_id + %{"name" => "message_id", "type" => "string"}, %{"name" => "file_id", "type" => "string"}, %{"name" => "belongs_to_id", "type" => "string"}, %{"name" => "inserted_at", "type" => "int64"}, diff --git a/lib/migration/typesense/photo.ex b/lib/migration/typesense/photo.ex index ae1abe5..fb66d48 100644 --- a/lib/migration/typesense/photo.ex +++ b/lib/migration/typesense/photo.ex @@ -17,12 +17,8 @@ defmodule Migration.Typesense.Photo do } }, %{"name" => "caption", "type" => "string", "optional" => true, "facet" => false}, - # "telegram:///" - # TODO: 不能再简单的 reset 了，reset 会导致数据丢失，应该合理 migrate 数据 - %{"name" => "url", "type" => "string"}, - # chat.id -> string + %{"name" => "file_id", "type" => "string"}, %{"name" => "belongs_to_id", "type" => "string"}, - # unix timestamp %{"name" => "inserted_at", "type" => "int64"} ], "default_sorting_field" => "inserted_at" diff --git a/lib/save_it/bot.ex b/lib/save_it/bot.ex index 60120a5..e402466 100644 --- a/lib/save_it/bot.ex +++ b/lib/save_it/bot.ex @@ -142,12 +142,11 @@ defmodule SaveIt.Bot do # end def handle( - {:command, :note, %{chat: chat, text: text, reply_to_message: reply_to_message}}, + {:command, :note, + %{message_id: message_id, chat: chat, text: text, reply_to_message: reply_to_message}}, _context ) when is_binary(text) do - Logger.debug("photo: #{inspect(reply_to_message.photo)}") - file_id = reply_to_message.photo |> List.last() |> Map.get(:file_id) case file_id do @@ -160,18 +159,17 @@ defmodule SaveIt.Bot do send_message(chat.id, "What note do you want to add?") note_content -> - # photo_url = photo_url(chat.id, file_id) - # PhotoService.update_photo!(photo_url, %{"note" => text}) - note = NoteService.create_note!(%{ content: note_content, + message_id: message_id, file_id: file_id, belongs_to_id: chat.id }) case note do nil -> send_message(chat.id, "Failed to add note.") + # TODO:nice_to_have: 添加一个 emoji 即可 _ -> send_message(chat.id, "Note added successfully.") end end @@ -187,20 +185,19 @@ defmodule SaveIt.Bot do # end # caption: nil -> find same photos - def handle({:message, %{chat: chat, caption: nil, photo: photos}}, ctx) do + def handle({:message, %{chat: chat, caption: nil, photo: photos}}, _ctx) do photo = List.last(photos) file = ExGram.get_file!(photo.file_id) photo_file_content = Telegram.download_file_content!(file.file_path) - bot_id = ctx.bot_info.id chat_id = chat.id typesense_photo = PhotoService.create_photo!(%{ image: Base.encode64(photo_file_content), caption: "", - url: photo_url(bot_id, file.file_id), + file_id: file.file_id, belongs_to_id: chat_id }) @@ -218,13 +215,12 @@ defmodule SaveIt.Bot do end # caption: contains /similar or /search -> search similar photos; otherwise, find same photos - def handle({:message, %{chat: chat, caption: caption, photo: photos}}, ctx) do + def handle({:message, %{chat: chat, caption: caption, photo: photos}}, _ctx) do photo = List.last(photos) file = ExGram.get_file!(photo.file_id) photo_file_content = Telegram.download_file_content!(file.file_path) - bot_id = ctx.bot_info.id chat_id = chat.id caption = @@ -238,7 +234,7 @@ defmodule SaveIt.Bot do PhotoService.create_photo!(%{ image: Base.encode64(photo_file_content), caption: caption, - url: photo_url(bot_id, file.file_id), + file_id: file.file_id, belongs_to_id: chat_id }) @@ -367,9 +363,24 @@ defmodule SaveIt.Bot do {:ok, nil} end - def handle({:edited_message, _msg}, _context) do - Logger.warning("this is an edited message, ignore it") - {:ok, nil} + def handle({:edited_message, msg}, _context) do + %{message_id: message_id, chat: chat, text: text} = msg + + edited_note_text = + case Regex.run(~r/\/note\s+(.*)/, text) do + [_, edited_note_text] -> edited_note_text + _ -> nil + end + + case String.contains?(text, "/note") do + true -> + note = NoteService.get_note!(message_id, chat.id) + + NoteService.update_note!(note["id"], %{"content" => edited_note_text}) + + false -> + Logger.debug("edited message: #{inspect(_msg)}") + end end def handle({:update, _update}, _context) do @@ -382,19 +393,6 @@ defmodule SaveIt.Bot do {:ok, nil} end - defp pick_file_id_from_photo_url(photo_url) do - captures = - Regex.named_captures(~r"/files/(?\d+)/(?.+)", photo_url) - - if captures == nil do - Logger.error("Invalid photo URL: #{photo_url}") - nil - else - %{"file_id" => file_id} = captures - file_id - end - end - defp answer_photos(chat_id, nil) do send_message(chat_id, "No photos found.") end @@ -408,7 +406,7 @@ defmodule SaveIt.Bot do Enum.map(similar_photos, fn photo -> %ExGram.Model.InputMediaPhoto{ type: "photo", - media: pick_file_id_from_photo_url(photo["url"]), + media: photo["file_id"], caption: "Found photos", show_caption_above_media: true } @@ -474,7 +472,7 @@ defmodule SaveIt.Bot do case file_extension(file_name) do ext when ext in [".png", ".jpg", ".jpeg"] -> {:ok, msg} = ExGram.send_photo(chat_id, content) - bot_id = msg.from.id + file_id = get_file_id(msg) image_base64 = @@ -486,7 +484,7 @@ defmodule SaveIt.Bot do PhotoService.create_photo!(%{ image: image_base64, caption: file_name, - url: photo_url(bot_id, file_id), + file_id: file_id, belongs_to_id: chat_id }) @@ -535,12 +533,4 @@ defmodule SaveIt.Bot do """) end end - - defp photo_url(bot_id, file_id) do - proxy_url = Application.fetch_env!(:save_it, :web_url) <> "/telegram/files" - - encoded_bot_id = URI.encode(bot_id |> to_string()) - encoded_file_id = URI.encode(file_id) - "#{proxy_url}/#{encoded_bot_id}/#{encoded_file_id}" - end end diff --git a/lib/save_it/note_service.ex b/lib/save_it/note_service.ex index 6a742d7..16803b7 100644 --- a/lib/save_it/note_service.ex +++ b/lib/save_it/note_service.ex @@ -3,29 +3,61 @@ defmodule SaveIt.NoteService do alias SmallSdk.Typesense - def create_note!(%{ - content: content, - file_id: file_id, - belongs_to_id: belongs_to_id - }) do + def create_note!( + %{ + message_id: message_id, + belongs_to_id: belongs_to_id + } = note_params + ) do now_unix = DateTime.utc_now() |> DateTime.to_unix() note_create_input = - %{ - content: content, - file_id: file_id - } + note_params + |> Map.put(:message_id, Integer.to_string(message_id)) |> Map.put(:belongs_to_id, Integer.to_string(belongs_to_id)) |> Map.put(:inserted_at, now_unix) |> Map.put(:updated_at, now_unix) - doc = - Typesense.create_document!( + Typesense.create_document!( + "notes", + note_create_input + ) + end + + def update_note!(id, %{} = note_params) do + now_unix = DateTime.utc_now() |> DateTime.to_unix() + + note_update_input = + note_params + |> Map.put(:updated_at, now_unix) + + Typesense.update_document!( + "notes", + id, + note_update_input + ) + end + + def get_note!(message_id, chat_id) do + docs = + Typesense.search_documents!( "notes", - note_create_input + q: "*", + query_by: "content", + filter_by: "message_id:=#{message_id} && belongs_to_id:=#{chat_id}" ) - Logger.debug("doc: #{inspect(doc)}") - doc + case docs do + nil -> + nil + + [] -> + nil + + [doc | rest] -> + Logger.warning("Found multiple notes, skipping the rest: #{inspect(rest)}") + + doc + end end end diff --git a/lib/save_it/photo_service.ex b/lib/save_it/photo_service.ex index b7aa2d8..1325949 100644 --- a/lib/save_it/photo_service.ex +++ b/lib/save_it/photo_service.ex @@ -19,7 +19,7 @@ defmodule SaveIt.PhotoService do end def update_photo(photo) do - Typesense.update_document("photos", photo.id, photo) + Typesense.update_document!("photos", photo.id, photo) end def get_photo(photo_id) do diff --git a/lib/small_sdk/typesense.ex b/lib/small_sdk/typesense.ex index 24d400d..75b27ff 100644 --- a/lib/small_sdk/typesense.ex +++ b/lib/small_sdk/typesense.ex @@ -17,7 +17,7 @@ defmodule SmallSdk.Typesense do raise "Unauthorized" %Req.Response{status: 404} -> - raise "Not Found" + nil %Req.Response{status: 409} -> raise "Conflict" @@ -40,18 +40,39 @@ defmodule SmallSdk.Typesense do handle_response(res) end + def search_documents!(collection_name, opts) do + q = Keyword.get(opts, :q, "*") + query_by = Keyword.get(opts, :query_by, "") + filter_by = Keyword.get(opts, :filter_by, "") + + req = build_request("/collections/#{collection_name}/documents/search") + + {:ok, res} = + Req.get(req, + params: %{ + q: q, + query_by: query_by, + filter_by: filter_by + } + ) + + data = handle_response(res) + + data["hits"] |> Enum.map(&Map.get(&1, "document")) + end + def get_document(collection_name, document_id) do req = build_request("/collections/#{collection_name}/documents/#{document_id}") {:ok, res} = Req.get(req) - res.body + handle_response(res) end - def update_document(collection_name, document_id, update_input) do + def update_document!(collection_name, document_id, update_input) do req = build_request("/collections/#{collection_name}/documents/#{document_id}") {:ok, res} = Req.patch(req, json: update_input) - res.body + handle_response(res) end def create_search_key() do From 5c43f19c6c43536256ffee5115dd8cd3943ee037 Mon Sep 17 00:00:00 2001 From: TJ Date: Fri, 25 Oct 2024 18:48:22 +0900 Subject: [PATCH 4/4] fix: miss --- lib/save_it/bot.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/save_it/bot.ex b/lib/save_it/bot.ex index e402466..4a38b06 100644 --- a/lib/save_it/bot.ex +++ b/lib/save_it/bot.ex @@ -379,7 +379,7 @@ defmodule SaveIt.Bot do NoteService.update_note!(note["id"], %{"content" => edited_note_text}) false -> - Logger.debug("edited message: #{inspect(_msg)}") + Logger.info("edited message: #{inspect(text)}, ignore it") end end