adding algolia indexer for releaser

package.json

@@ -24,7 +24,6 @@
     "test": "yarn prepareTest && yarn build && ava",
     "bundle": "yarn build && pkg . --out-dir ./exec/",
     "release-db": "ts-node ./src/releaser/sync/sync-database.ts"
-
   },
   "devDependencies": {
     "@types/ethereum-checksum-address": "^0.0.0",
@@ -36,24 +35,25 @@
     "typescript": "^4.7.4"
   },
   "dependencies": {
+    "@octokit/auth-token": "^3.0.0",
+    "@octokit/rest": "^19.0.3",
+    "@types/moment": "^2.13.0",
+    "@types/sqlite3": "^3.1.8",
     "@uniswap/token-lists": "^1.0.0-beta.30",
+    "algoliasearch": "^4.15.0",
     "axios": "^0.27.2",
     "csv-parse": "^5.3.0",
+    "dotenv": "^16.0.1",
     "ethereum-checksum-address": "^0.0.8",
     "ethers": "^5.7.2",
     "jsonschema": "^1.4.1",
-    "shelljs": "^0.8.5",
-    "uuid": "^8.3.2",
-    "@octokit/auth-token": "^3.0.0",
-    "@octokit/rest": "^19.0.3",
-    "@types/moment": "^2.13.0",
-    "@types/sqlite3": "^3.1.8",
-    "dotenv": "^16.0.1",
     "ksuid": "^3.0.0",
     "moment": "^2.29.4",
     "octokit": "^2.0.5",
     "promised-sqlite3": "^1.2.0",
-    "sqlite3": "^5.0.11"
+    "shelljs": "^0.8.5",
+    "sqlite3": "^5.0.11",
+    "uuid": "^8.3.2"
   },
   "packageManager": "[email protected]",
   "repository": {

src/db/asset.ts

@@ -7,11 +7,11 @@ import { ETHEREUM_ASSETS, POLYGON_ASSETS } from "./assets.json";
 type AssetInfoCallback = (assetInfo: Asset) => Promise<void>;
 type AssetMappingInfoCallback = (assetMapping: AssetMapping) => Promise<void>;
 
-export async function assetsForEach(callback: AssetInfoCallback, complete?: () => Promise<void>) {
+export async function assetsForEach(callback: AssetInfoCallback, complete?: () => Promise<void>, dir?: string) {
   try {
-    const networks = await getNetworks();
+    const networks = await getNetworks(dir);
     for (const network of networks) {
-      const assets = await getAssetsForNetwork(network.networkCode);
+      const assets = await getAssetsForNetwork(network.networkCode, dir);
       for (const asset of assets) {
         await callback(asset);
       }

src/db/network.ts

@@ -4,9 +4,9 @@ import { MOCK_NETWORKS } from "./networks.json";
 
 type NetworkInfoCallback = (networkInfo: Network) => Promise<void>;
 
-export async function networksForEach(callback: NetworkInfoCallback, complete?: () => Promise<void>) {
+export async function networksForEach(callback: NetworkInfoCallback, complete?: () => Promise<void>, dir?: string) {
   try {
-    const networks = await getNetworks();
+    const networks = await getNetworks(dir);
     await Promise.all(networks.map(async network => {
       return callback(network);
     }));

src/index.ts

@@ -9,6 +9,7 @@ export * from './trustwallet';
 export * from './utils';
 export * from './validate';
 export * from './db';
+export * from './releaser';
 
 // const base = '/Users/ap/ama_dev/map3/indexer/tmp';
 // ingestTokenList(`${base}/kq.tokenlist.json`, `${base}/map3xyz-assets/networks/polygon/assets/polygon-tokenlist`, 'foo', 'polygon-manual-list' )

src/releaser/algolia/index.ts

@@ -0,0 +1,125 @@
+import algoliasearch, { SearchIndex } from "algoliasearch";
+import { networksForEach, assetsForEach } from "../../db";
+import { ObjectType, Description, ChainObject, Network, Asset } from "../../model";
+
+const updateOptions = {
+  createIfNotExists: true,
+};
+
+declare interface IndexedAsset {
+  objectID: string;
+  address?: string;
+  networkCode: string;
+  type: ObjectType;
+  name: string;
+  symbol: string;
+  description: Description | {};
+  logoUrl: string | null;
+}
+
+function transform({
+  id: objectID,
+  networkCode,
+  type,
+  name,
+  symbol,
+  description,
+  logo,
+}: ChainObject): IndexedAsset {
+  const logoParsed = logo;
+  const logoUrl = logoParsed?.svg || logoParsed?.png || null;
+  description = description || {};
+
+  return {
+    objectID,
+    networkCode,
+    type,
+    name,
+    symbol,
+    description,
+    logoUrl,
+  };
+}
+
+export async function updateAlgoliaSearch(
+  directory: string,
+  batchSize: Number = 250
+): Promise<void> {
+    if(!process.env.ALGOLIA_APP_ID) {
+        throw new Error('Missing Algolia App ID');
+    }
+
+    if(!process.env.ALGOLIA_API_KEY) {
+        throw new Error('Missing Algolia API Key');
+    }
+
+    const client = algoliasearch(process.env.ALGOLIA_APP_ID, 
+        process.env.ALGOLIA_API_KEY);
+    const assets = client.initIndex("assets");
+
+    try {
+      await assets.setSettings({
+        searchableAttributes: [
+          "objectID",
+          "unordered(name)",
+          "symbol",
+          "unordered(description)",
+          "address",
+        ],
+        attributesForFaceting: ["type", "networkCode"],
+      });
+
+      await updateNetworks(assets, batchSize, directory);
+      await updateAssets(assets, batchSize, directory);
+
+      console.log('Updated Algolia Search')
+      Promise.resolve();
+    } catch (err: any) {
+      console.error('Error with Algolia search', err);
+      throw err;
+    }
+}
+
+async function updateNetworks(index: SearchIndex, batchSize: Number, dir: string) {
+  let buffer: IndexedAsset[] = [];
+  async function updateNetwork(networkInfo: Network) {
+    if (buffer.length < batchSize) {
+      buffer.push(transform(networkInfo));
+    } else {
+      await index.partialUpdateObjects(buffer, updateOptions);
+      buffer = [];
+    }
+  }
+  async function completeUpdate() {
+    if (buffer.length > 0) {
+      const response = await index.partialUpdateObjects(buffer, updateOptions);
+    }
+  }
+
+  await networksForEach(updateNetwork, completeUpdate, dir);
+  return Promise.resolve();
+}
+
+async function updateAssets(index: SearchIndex, batchSize: Number, dir: string) {
+  let buffer: IndexedAsset[] = [];
+  async function updateAsset(assetInfo: Asset) {
+    if (buffer.length < batchSize) {
+      const asset = transform(assetInfo);
+      buffer.push({ ...asset, address: assetInfo.address });
+    } else {
+      await index.partialUpdateObjects(buffer, updateOptions);
+      buffer = [];
+    }
+
+    return Promise.resolve();
+  }
+  async function completeUpdate() {
+    if (buffer.length > 0) {
+      await index.partialUpdateObjects(buffer, updateOptions);
+    }
+    return Promise.resolve();
+  }
+
+  await assetsForEach(updateAsset, completeUpdate, dir);
+  return Promise.resolve();
+}

src/releaser/sync/sync-database.ts → src/releaser/db/sync-database.ts

@@ -15,13 +15,13 @@ const ASSET_REPO_OWNER = "map3xyz";
 const ASSET_REPO_NAME = "assets";
 
 dotenv.config();
-createAssetDb();
+
 interface AssetMapExt extends AssetMap {
   fromId: string;
   toId: string;
 }
 
-async function createAssetDb(assetRepo: string = "/tmp/assets", assetDb: string = "/tmp/assets.db"): Promise<void> {
+export async function createAssetDb(assetRepo: string = "/tmp/assets", assetDb: string = "/tmp/assets.db"): Promise<void> {
   if(!process.env.GITHUB_TOKEN) {
     throw new Error("GITHUB_TOKEN not set");
   }

src/releaser/index.ts

@@ -0,0 +1,9 @@
+import { updateAlgoliaSearch } from "./algolia";
+import { createAssetDb } from "./db/sync-database";
+
+export async function releaseDb(directory = '/tmp/assets') {
+    return Promise.all([
+        createAssetDb(directory), 
+        updateAlgoliaSearch(directory)
+    ]) 
+}

yarn.lock

@@ -2,6 +2,110 @@
 # yarn lockfile v1
 
 
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.15.0.tgz#84f12aec6b6aa69542a3bfd3a4ba458ed2cc8230"
+  integrity sha512-uxxFhTWh4JJDb2+FFSmNMfEQ8p9o2vjSpU7iW007QX3OvqljPPN68lk3bpZVaG8pwr5MU1DqpkZ71FcQdVTjgQ==
+  dependencies:
+    "@algolia/cache-common" "4.15.0"
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/cache-common/-/cache-common-4.15.0.tgz#a198098c4b8fa6ef661879ec22d2a2d1ad77d2bb"
+  integrity sha512-Me3PbI4QurAM+3D+htIE0l1xt6+bl/18SG6Wc7bPQEZAtN7DTGz22HqhKNyLF2lR/cOfpaH7umXZlZEhIHf7gQ==
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/cache-in-memory/-/cache-in-memory-4.15.0.tgz#77cac4db36a0aa0837f7a7ceb760188191e35268"
+  integrity sha512-B9mg1wd7CKMfpkbiTQ8KlcKkH6ut/goVaI6XmDCUczOOqeuZlV34tuEi7o3Xo1j66KWr/d9pMjjGYcoVPCVeOA==
+  dependencies:
+    "@algolia/cache-common" "4.15.0"
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/client-account/-/client-account-4.15.0.tgz#8e0723052169665b4449dc2f8bcf3075feb6a424"
+  integrity sha512-8wqI33HRZy5ydfFt6F5vMhtkOiAUhVfSCYXx4U3Go5RALqWLgVUp6wzOo0mr1z08POCkHDpbQMQvyayb1CZ/kw==
+  dependencies:
+    "@algolia/client-common" "4.15.0"
+    "@algolia/client-search" "4.15.0"
+    "@algolia/transporter" "4.15.0"
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/client-analytics/-/client-analytics-4.15.0.tgz#6b8fe450e1bba114b0d0598cbf9acac482798a36"
+  integrity sha512-jrPjEeNEIIQKeA1XCZXx3f3aybtwF7wjYlnfHbLARuZ9AuHzimOKjX0ZwqvMmvTsHivpcZ2rqY+j1E8HoH1ELA==
+  dependencies:
+    "@algolia/client-common" "4.15.0"
+    "@algolia/client-search" "4.15.0"
+    "@algolia/requester-common" "4.15.0"
+    "@algolia/transporter" "4.15.0"
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/client-common/-/client-common-4.15.0.tgz#27dd9441aedf481736696d519e55ea8e2f5a4432"
+  integrity sha512-PlsJMObZuYw4JlG5EhYv1PHDOv7n5mD5PzqFyoNfSOYaEPRZepa3W579ya29yOu3FZ0VGMNJmB7Q5v/+/fwvIw==
+  dependencies:
+    "@algolia/requester-common" "4.15.0"
+    "@algolia/transporter" "4.15.0"
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/client-personalization/-/client-personalization-4.15.0.tgz#6f10eda827d2607ab6c2341464cd35107bf8cf99"
+  integrity sha512-Bf0bhRAiNL9LWurzyHRH8UBi4fDt3VbCNkInxVngKQT1uCZWXecwoPWGhcSSpdanBqFJA/1WBt+BWx7a50Bhlg==
+  dependencies:
+    "@algolia/client-common" "4.15.0"
+    "@algolia/requester-common" "4.15.0"
+    "@algolia/transporter" "4.15.0"
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/client-search/-/client-search-4.15.0.tgz#2d849faae7943fcc983ac923eac767666a9e6a9a"
+  integrity sha512-dTwZD4u53WdmexnMcoO2Qd/+YCP3ESXKOtD2MryQ1a9dHwB2Y3Qob0kyS1PG82idwM3enbznvscI9Sf4o9PUWQ==
+  dependencies:
+    "@algolia/client-common" "4.15.0"
+    "@algolia/requester-common" "4.15.0"
+    "@algolia/transporter" "4.15.0"
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/logger-common/-/logger-common-4.15.0.tgz#a2cf3d3abbdd00594006164302600ba46d75059f"
+  integrity sha512-D8OFwn/HpvQz66goIcjxOKsYBMuxiruxJ3cA/bnc0EiDvSA2P2z6bNQWgS5gbstuTZIJmbhr+53NyOxFkmMNAA==
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/logger-console/-/logger-console-4.15.0.tgz#8a0948b0c16ad546af9dd14b9021f21f42737c97"
+  integrity sha512-pQOvVaRSEJQJRXKTnxEA6nN1hipSQadJJ4einw0nIlfMOGZh/kps1ybh8vRUlUGyfEuN/3dyFs0W3Ac7hIItlg==
+  dependencies:
+    "@algolia/logger-common" "4.15.0"
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.15.0.tgz#38b5956d01408ad4291d89915df921ff8534cca6"
+  integrity sha512-va186EfALF+6msYZXaoBSxcnFCg3SoWJ+uv1yMyhQRJRe7cZSHWSVT3s40vmar90gxlBu80KMVwVlsvJhJv6ew==
+  dependencies:
+    "@algolia/requester-common" "4.15.0"
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/requester-common/-/requester-common-4.15.0.tgz#c68ad3dccc1de71b5be9b08a07e2baf58ec49d82"
+  integrity sha512-w0UUzxElbo4hrKg4QP/jiXDNbIJuAthxdlkos9nS8KAPK2XI3R9BlUjLz/ZVs4F9TDGI0mhjrNHhZ12KXcoyhg==
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/requester-node-http/-/requester-node-http-4.15.0.tgz#02f841586e620c7b4e4e555f315cd52dd815f330"
+  integrity sha512-eeEOhFtgwKcgAlKAZpgBRZJ0ILSEBCXxZ9uwfVWPD24W1b6z08gVoTJ6J7lCeCnJmudg+tMElDnGzHkjup9CJA==
+  dependencies:
+    "@algolia/requester-common" "4.15.0"
+
+"@algolia/[email protected]":
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/@algolia/transporter/-/transporter-4.15.0.tgz#c65c512206c66aadc2897337220ae5454001967e"
+  integrity sha512-JoWR+ixG3EmA0UPntQFN/FV5TasYcYu93d5+oKzHFeZ6Z7rtW5Im9iy/Oh/ggk1AAN5fTdqKewtbBpdaYDbKsQ==
+  dependencies:
+    "@algolia/cache-common" "4.15.0"
+    "@algolia/logger-common" "4.15.0"
+    "@algolia/requester-common" "4.15.0"
+
 "@babel/code-frame@^7.0.0":
   version "7.18.6"
   resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.18.6.tgz#3b25d38c89600baa2dcc219edfa88a74eb2c427a"
@@ -888,6 +992,26 @@ aggregate-error@^4.0.0:
     clean-stack "^4.0.0"
     indent-string "^5.0.0"
 
+algoliasearch@^4.15.0:
+  version "4.15.0"
+  resolved "https://registry.yarnpkg.com/algoliasearch/-/algoliasearch-4.15.0.tgz#8279576f06667a1d0705e8c22a17daa8e707b469"
+  integrity sha512-+vgKQF5944dYsz9zhKk07JbOYeNdKisoD5GeG0woBL3nLzbn2a+nGwki60DXg7CXvaFXBcTXyJG4C+VaBVd44g==
+  dependencies:
+    "@algolia/cache-browser-local-storage" "4.15.0"
+    "@algolia/cache-common" "4.15.0"
+    "@algolia/cache-in-memory" "4.15.0"
+    "@algolia/client-account" "4.15.0"
+    "@algolia/client-analytics" "4.15.0"
+    "@algolia/client-common" "4.15.0"
+    "@algolia/client-personalization" "4.15.0"
+    "@algolia/client-search" "4.15.0"
+    "@algolia/logger-common" "4.15.0"
+    "@algolia/logger-console" "4.15.0"
+    "@algolia/requester-browser-xhr" "4.15.0"
+    "@algolia/requester-common" "4.15.0"
+    "@algolia/requester-node-http" "4.15.0"
+    "@algolia/transporter" "4.15.0"
+
 ansi-regex@^2.0.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df"