From cf19db893e76474b03f5c49cf1ebe77a51bba332 Mon Sep 17 00:00:00 2001 From: Amadeo Pellicce Date: Fri, 10 Mar 2023 11:51:54 -0800 Subject: [PATCH] adding algolia indexer for releaser --- package.json | 18 +-- src/db/asset.ts | 6 +- src/db/network.ts | 4 +- src/index.ts | 1 + src/releaser/algolia/index.ts | 125 +++++++++++++++++++++ src/releaser/{sync => db}/sync-database.ts | 4 +- src/releaser/index.ts | 9 ++ yarn.lock | 124 ++++++++++++++++++++ 8 files changed, 275 insertions(+), 16 deletions(-) create mode 100644 src/releaser/algolia/index.ts rename src/releaser/{sync => db}/sync-database.ts (98%) create mode 100644 src/releaser/index.ts diff --git a/package.json b/package.json index 5154ffd..809b809 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,6 @@ "test": "yarn prepareTest && yarn build && ava", "bundle": "yarn build && pkg . --out-dir ./exec/", "release-db": "ts-node ./src/releaser/sync/sync-database.ts" - }, "devDependencies": { "@types/ethereum-checksum-address": "^0.0.0", @@ -36,24 +35,25 @@ "typescript": "^4.7.4" }, "dependencies": { + "@octokit/auth-token": "^3.0.0", + "@octokit/rest": "^19.0.3", + "@types/moment": "^2.13.0", + "@types/sqlite3": "^3.1.8", "@uniswap/token-lists": "^1.0.0-beta.30", + "algoliasearch": "^4.15.0", "axios": "^0.27.2", "csv-parse": "^5.3.0", + "dotenv": "^16.0.1", "ethereum-checksum-address": "^0.0.8", "ethers": "^5.7.2", "jsonschema": "^1.4.1", - "shelljs": "^0.8.5", - "uuid": "^8.3.2", - "@octokit/auth-token": "^3.0.0", - "@octokit/rest": "^19.0.3", - "@types/moment": "^2.13.0", - "@types/sqlite3": "^3.1.8", - "dotenv": "^16.0.1", "ksuid": "^3.0.0", "moment": "^2.29.4", "octokit": "^2.0.5", "promised-sqlite3": "^1.2.0", - "sqlite3": "^5.0.11" + "shelljs": "^0.8.5", + "sqlite3": "^5.0.11", + "uuid": "^8.3.2" }, "packageManager": "yarn@3.2.1", "repository": { diff --git a/src/db/asset.ts b/src/db/asset.ts index 33f1884..99352dd 100644 --- a/src/db/asset.ts +++ b/src/db/asset.ts @@ -7,11 +7,11 @@ import { ETHEREUM_ASSETS, POLYGON_ASSETS } from "./assets.json"; type AssetInfoCallback = (assetInfo: Asset) => Promise; type AssetMappingInfoCallback = (assetMapping: AssetMapping) => Promise; -export async function assetsForEach(callback: AssetInfoCallback, complete?: () => Promise) { +export async function assetsForEach(callback: AssetInfoCallback, complete?: () => Promise, dir?: string) { try { - const networks = await getNetworks(); + const networks = await getNetworks(dir); for (const network of networks) { - const assets = await getAssetsForNetwork(network.networkCode); + const assets = await getAssetsForNetwork(network.networkCode, dir); for (const asset of assets) { await callback(asset); } diff --git a/src/db/network.ts b/src/db/network.ts index 9db890f..8d18b00 100644 --- a/src/db/network.ts +++ b/src/db/network.ts @@ -4,9 +4,9 @@ import { MOCK_NETWORKS } from "./networks.json"; type NetworkInfoCallback = (networkInfo: Network) => Promise; -export async function networksForEach(callback: NetworkInfoCallback, complete?: () => Promise) { +export async function networksForEach(callback: NetworkInfoCallback, complete?: () => Promise, dir?: string) { try { - const networks = await getNetworks(); + const networks = await getNetworks(dir); await Promise.all(networks.map(async network => { return callback(network); })); diff --git a/src/index.ts b/src/index.ts index 5b8f048..fe49c8b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ export * from './trustwallet'; export * from './utils'; export * from './validate'; export * from './db'; +export * from './releaser'; // const base = '/Users/ap/ama_dev/map3/indexer/tmp'; // ingestTokenList(`${base}/kq.tokenlist.json`, `${base}/map3xyz-assets/networks/polygon/assets/polygon-tokenlist`, 'foo', 'polygon-manual-list' ) diff --git a/src/releaser/algolia/index.ts b/src/releaser/algolia/index.ts new file mode 100644 index 0000000..c5211a8 --- /dev/null +++ b/src/releaser/algolia/index.ts @@ -0,0 +1,125 @@ +import algoliasearch, { SearchIndex } from "algoliasearch"; +import { networksForEach, assetsForEach } from "../../db"; +import { ObjectType, Description, ChainObject, Network, Asset } from "../../model"; + +const updateOptions = { + createIfNotExists: true, +}; + +declare interface IndexedAsset { + objectID: string; + address?: string; + networkCode: string; + type: ObjectType; + name: string; + symbol: string; + description: Description | {}; + logoUrl: string | null; +} + +function transform({ + id: objectID, + networkCode, + type, + name, + symbol, + description, + logo, +}: ChainObject): IndexedAsset { + const logoParsed = logo; + const logoUrl = logoParsed?.svg || logoParsed?.png || null; + description = description || {}; + + return { + objectID, + networkCode, + type, + name, + symbol, + description, + logoUrl, + }; +} + +export async function updateAlgoliaSearch( + directory: string, + batchSize: Number = 250 +): Promise { + if(!process.env.ALGOLIA_APP_ID) { + throw new Error('Missing Algolia App ID'); + } + + if(!process.env.ALGOLIA_API_KEY) { + throw new Error('Missing Algolia API Key'); + } + + const client = algoliasearch(process.env.ALGOLIA_APP_ID, + process.env.ALGOLIA_API_KEY); + const assets = client.initIndex("assets"); + + try { + await assets.setSettings({ + searchableAttributes: [ + "objectID", + "unordered(name)", + "symbol", + "unordered(description)", + "address", + ], + attributesForFaceting: ["type", "networkCode"], + }); + + await updateNetworks(assets, batchSize, directory); + await updateAssets(assets, batchSize, directory); + + console.log('Updated Algolia Search') + Promise.resolve(); + } catch (err: any) { + console.error('Error with Algolia search', err); + throw err; + } +} + +async function updateNetworks(index: SearchIndex, batchSize: Number, dir: string) { + let buffer: IndexedAsset[] = []; + async function updateNetwork(networkInfo: Network) { + if (buffer.length < batchSize) { + buffer.push(transform(networkInfo)); + } else { + await index.partialUpdateObjects(buffer, updateOptions); + buffer = []; + } + } + async function completeUpdate() { + if (buffer.length > 0) { + const response = await index.partialUpdateObjects(buffer, updateOptions); + } + } + + await networksForEach(updateNetwork, completeUpdate, dir); + return Promise.resolve(); +} + +async function updateAssets(index: SearchIndex, batchSize: Number, dir: string) { + let buffer: IndexedAsset[] = []; + async function updateAsset(assetInfo: Asset) { + if (buffer.length < batchSize) { + const asset = transform(assetInfo); + buffer.push({ ...asset, address: assetInfo.address }); + } else { + await index.partialUpdateObjects(buffer, updateOptions); + buffer = []; + } + + return Promise.resolve(); + } + async function completeUpdate() { + if (buffer.length > 0) { + await index.partialUpdateObjects(buffer, updateOptions); + } + return Promise.resolve(); + } + + await assetsForEach(updateAsset, completeUpdate, dir); + return Promise.resolve(); +} diff --git a/src/releaser/sync/sync-database.ts b/src/releaser/db/sync-database.ts similarity index 98% rename from src/releaser/sync/sync-database.ts rename to src/releaser/db/sync-database.ts index c08e4e6..87f5350 100644 --- a/src/releaser/sync/sync-database.ts +++ b/src/releaser/db/sync-database.ts @@ -15,13 +15,13 @@ const ASSET_REPO_OWNER = "map3xyz"; const ASSET_REPO_NAME = "assets"; dotenv.config(); -createAssetDb(); + interface AssetMapExt extends AssetMap { fromId: string; toId: string; } -async function createAssetDb(assetRepo: string = "/tmp/assets", assetDb: string = "/tmp/assets.db"): Promise { +export async function createAssetDb(assetRepo: string = "/tmp/assets", assetDb: string = "/tmp/assets.db"): Promise { if(!process.env.GITHUB_TOKEN) { throw new Error("GITHUB_TOKEN not set"); } diff --git a/src/releaser/index.ts b/src/releaser/index.ts new file mode 100644 index 0000000..b5227fe --- /dev/null +++ b/src/releaser/index.ts @@ -0,0 +1,9 @@ +import { updateAlgoliaSearch } from "./algolia"; +import { createAssetDb } from "./db/sync-database"; + +export async function releaseDb(directory = '/tmp/assets') { + return Promise.all([ + createAssetDb(directory), + updateAlgoliaSearch(directory) + ]) +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index d0f1f45..9927199 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,110 @@ # yarn lockfile v1 +"@algolia/cache-browser-local-storage@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.15.0.tgz#84f12aec6b6aa69542a3bfd3a4ba458ed2cc8230" + integrity sha512-uxxFhTWh4JJDb2+FFSmNMfEQ8p9o2vjSpU7iW007QX3OvqljPPN68lk3bpZVaG8pwr5MU1DqpkZ71FcQdVTjgQ== + dependencies: + "@algolia/cache-common" "4.15.0" + +"@algolia/cache-common@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/cache-common/-/cache-common-4.15.0.tgz#a198098c4b8fa6ef661879ec22d2a2d1ad77d2bb" + integrity sha512-Me3PbI4QurAM+3D+htIE0l1xt6+bl/18SG6Wc7bPQEZAtN7DTGz22HqhKNyLF2lR/cOfpaH7umXZlZEhIHf7gQ== + +"@algolia/cache-in-memory@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/cache-in-memory/-/cache-in-memory-4.15.0.tgz#77cac4db36a0aa0837f7a7ceb760188191e35268" + integrity sha512-B9mg1wd7CKMfpkbiTQ8KlcKkH6ut/goVaI6XmDCUczOOqeuZlV34tuEi7o3Xo1j66KWr/d9pMjjGYcoVPCVeOA== + dependencies: + "@algolia/cache-common" "4.15.0" + +"@algolia/client-account@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/client-account/-/client-account-4.15.0.tgz#8e0723052169665b4449dc2f8bcf3075feb6a424" + integrity sha512-8wqI33HRZy5ydfFt6F5vMhtkOiAUhVfSCYXx4U3Go5RALqWLgVUp6wzOo0mr1z08POCkHDpbQMQvyayb1CZ/kw== + dependencies: + "@algolia/client-common" "4.15.0" + "@algolia/client-search" "4.15.0" + "@algolia/transporter" "4.15.0" + +"@algolia/client-analytics@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/client-analytics/-/client-analytics-4.15.0.tgz#6b8fe450e1bba114b0d0598cbf9acac482798a36" + integrity sha512-jrPjEeNEIIQKeA1XCZXx3f3aybtwF7wjYlnfHbLARuZ9AuHzimOKjX0ZwqvMmvTsHivpcZ2rqY+j1E8HoH1ELA== + dependencies: + "@algolia/client-common" "4.15.0" + "@algolia/client-search" "4.15.0" + "@algolia/requester-common" "4.15.0" + "@algolia/transporter" "4.15.0" + +"@algolia/client-common@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/client-common/-/client-common-4.15.0.tgz#27dd9441aedf481736696d519e55ea8e2f5a4432" + integrity sha512-PlsJMObZuYw4JlG5EhYv1PHDOv7n5mD5PzqFyoNfSOYaEPRZepa3W579ya29yOu3FZ0VGMNJmB7Q5v/+/fwvIw== + dependencies: + "@algolia/requester-common" "4.15.0" + "@algolia/transporter" "4.15.0" + +"@algolia/client-personalization@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/client-personalization/-/client-personalization-4.15.0.tgz#6f10eda827d2607ab6c2341464cd35107bf8cf99" + integrity sha512-Bf0bhRAiNL9LWurzyHRH8UBi4fDt3VbCNkInxVngKQT1uCZWXecwoPWGhcSSpdanBqFJA/1WBt+BWx7a50Bhlg== + dependencies: + "@algolia/client-common" "4.15.0" + "@algolia/requester-common" "4.15.0" + "@algolia/transporter" "4.15.0" + +"@algolia/client-search@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/client-search/-/client-search-4.15.0.tgz#2d849faae7943fcc983ac923eac767666a9e6a9a" + integrity sha512-dTwZD4u53WdmexnMcoO2Qd/+YCP3ESXKOtD2MryQ1a9dHwB2Y3Qob0kyS1PG82idwM3enbznvscI9Sf4o9PUWQ== + dependencies: + "@algolia/client-common" "4.15.0" + "@algolia/requester-common" "4.15.0" + "@algolia/transporter" "4.15.0" + +"@algolia/logger-common@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/logger-common/-/logger-common-4.15.0.tgz#a2cf3d3abbdd00594006164302600ba46d75059f" + integrity sha512-D8OFwn/HpvQz66goIcjxOKsYBMuxiruxJ3cA/bnc0EiDvSA2P2z6bNQWgS5gbstuTZIJmbhr+53NyOxFkmMNAA== + +"@algolia/logger-console@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/logger-console/-/logger-console-4.15.0.tgz#8a0948b0c16ad546af9dd14b9021f21f42737c97" + integrity sha512-pQOvVaRSEJQJRXKTnxEA6nN1hipSQadJJ4einw0nIlfMOGZh/kps1ybh8vRUlUGyfEuN/3dyFs0W3Ac7hIItlg== + dependencies: + "@algolia/logger-common" "4.15.0" + +"@algolia/requester-browser-xhr@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.15.0.tgz#38b5956d01408ad4291d89915df921ff8534cca6" + integrity sha512-va186EfALF+6msYZXaoBSxcnFCg3SoWJ+uv1yMyhQRJRe7cZSHWSVT3s40vmar90gxlBu80KMVwVlsvJhJv6ew== + dependencies: + "@algolia/requester-common" "4.15.0" + +"@algolia/requester-common@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/requester-common/-/requester-common-4.15.0.tgz#c68ad3dccc1de71b5be9b08a07e2baf58ec49d82" + integrity sha512-w0UUzxElbo4hrKg4QP/jiXDNbIJuAthxdlkos9nS8KAPK2XI3R9BlUjLz/ZVs4F9TDGI0mhjrNHhZ12KXcoyhg== + +"@algolia/requester-node-http@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/requester-node-http/-/requester-node-http-4.15.0.tgz#02f841586e620c7b4e4e555f315cd52dd815f330" + integrity sha512-eeEOhFtgwKcgAlKAZpgBRZJ0ILSEBCXxZ9uwfVWPD24W1b6z08gVoTJ6J7lCeCnJmudg+tMElDnGzHkjup9CJA== + dependencies: + "@algolia/requester-common" "4.15.0" + +"@algolia/transporter@4.15.0": + version "4.15.0" + resolved "https://registry.yarnpkg.com/@algolia/transporter/-/transporter-4.15.0.tgz#c65c512206c66aadc2897337220ae5454001967e" + integrity sha512-JoWR+ixG3EmA0UPntQFN/FV5TasYcYu93d5+oKzHFeZ6Z7rtW5Im9iy/Oh/ggk1AAN5fTdqKewtbBpdaYDbKsQ== + dependencies: + "@algolia/cache-common" "4.15.0" + "@algolia/logger-common" "4.15.0" + "@algolia/requester-common" "4.15.0" + "@babel/code-frame@^7.0.0": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.18.6.tgz#3b25d38c89600baa2dcc219edfa88a74eb2c427a" @@ -888,6 +992,26 @@ aggregate-error@^4.0.0: clean-stack "^4.0.0" indent-string "^5.0.0" +algoliasearch@^4.15.0: + version "4.15.0" + resolved "https://registry.yarnpkg.com/algoliasearch/-/algoliasearch-4.15.0.tgz#8279576f06667a1d0705e8c22a17daa8e707b469" + integrity sha512-+vgKQF5944dYsz9zhKk07JbOYeNdKisoD5GeG0woBL3nLzbn2a+nGwki60DXg7CXvaFXBcTXyJG4C+VaBVd44g== + dependencies: + "@algolia/cache-browser-local-storage" "4.15.0" + "@algolia/cache-common" "4.15.0" + "@algolia/cache-in-memory" "4.15.0" + "@algolia/client-account" "4.15.0" + "@algolia/client-analytics" "4.15.0" + "@algolia/client-common" "4.15.0" + "@algolia/client-personalization" "4.15.0" + "@algolia/client-search" "4.15.0" + "@algolia/logger-common" "4.15.0" + "@algolia/logger-console" "4.15.0" + "@algolia/requester-browser-xhr" "4.15.0" + "@algolia/requester-common" "4.15.0" + "@algolia/requester-node-http" "4.15.0" + "@algolia/transporter" "4.15.0" + ansi-regex@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df"