Merge pull request #40 from map3xyz/db-utils

Adding Db utils to release database and update algolia index

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@map3xyz/assets-helper",
-  "version": "1.0.180",
+  "version": "1.0.188",
   "description": "A library for maintaining the assets repo.",
   "author": "pellicceama",
   "keywords": [
@@ -18,11 +18,12 @@
   "scripts": {
     "index": "ts-node ./src/index.ts",
     "clean": "rimraf ./dist/ ./exec/",
-    "build": "yarn clean && tsc",
+    "build": "yarn clean && tsc && cp -r ./src/releaser/schema ./dist/releaser/schema",
     "preparePublish": "yarn build && npm version patch",
     "prepareTest": "yarn ts-node ./src/cache.ts",
     "test": "yarn prepareTest && yarn build && ava",
-    "bundle": "yarn build && pkg . --out-dir ./exec/"
+    "bundle": "yarn build && pkg . --out-dir ./exec/",
+    "release-db": "ts-node ./src/releaser/sync/sync-database.ts"
   },
   "devDependencies": {
     "@types/ethereum-checksum-address": "^0.0.0",
@@ -34,13 +35,24 @@
     "typescript": "^4.7.4"
   },
   "dependencies": {
+    "@octokit/auth-token": "^3.0.0",
+    "@octokit/rest": "^19.0.3",
+    "@types/moment": "^2.13.0",
+    "@types/sqlite3": "^3.1.8",
     "@uniswap/token-lists": "^1.0.0-beta.30",
+    "algoliasearch": "^4.15.0",
     "axios": "^0.27.2",
     "csv-parse": "^5.3.0",
+    "dotenv": "^16.0.1",
     "ethereum-checksum-address": "^0.0.8",
     "ethers": "^5.7.2",
     "jsonschema": "^1.4.1",
+    "ksuid": "^3.0.0",
+    "moment": "^2.29.4",
+    "octokit": "^2.0.5",
+    "promised-sqlite3": "^1.2.0",
     "shelljs": "^0.8.5",
+    "sqlite3": "^5.0.11",
     "uuid": "^8.3.2"
   },
   "packageManager": "[email protected]",

src/db/asset.ts

@@ -7,11 +7,11 @@ import { ETHEREUM_ASSETS, POLYGON_ASSETS } from "./assets.json";
 type AssetInfoCallback = (assetInfo: Asset) => Promise<void>;
 type AssetMappingInfoCallback = (assetMapping: AssetMapping) => Promise<void>;
 
-export async function assetsForEach(callback: AssetInfoCallback, complete?: () => Promise<void>) {
+export async function assetsForEach(callback: AssetInfoCallback, complete?: () => Promise<void>, dir?: string) {
   try {
-    const networks = await getNetworks();
+    const networks = await getNetworks(dir);
     for (const network of networks) {
-      const assets = await getAssetsForNetwork(network.networkCode);
+      const assets = await getAssetsForNetwork(network.networkCode, dir);
       for (const asset of assets) {
         await callback(asset);
       }

src/db/network.ts

@@ -4,9 +4,9 @@ import { MOCK_NETWORKS } from "./networks.json";
 
 type NetworkInfoCallback = (networkInfo: Network) => Promise<void>;
 
-export async function networksForEach(callback: NetworkInfoCallback, complete?: () => Promise<void>) {
+export async function networksForEach(callback: NetworkInfoCallback, complete?: () => Promise<void>, dir?: string) {
   try {
-    const networks = await getNetworks();
+    const networks = await getNetworks(dir);
     await Promise.all(networks.map(async network => {
       return callback(network);
     }));

src/index.ts

@@ -9,6 +9,7 @@ export * from './trustwallet';
 export * from './utils';
 export * from './validate';
 export * from './db';
+export * from './releaser';
 
 // const base = '/Users/ap/ama_dev/map3/indexer/tmp';
 // ingestTokenList(`${base}/kq.tokenlist.json`, `${base}/map3xyz-assets/networks/polygon/assets/polygon-tokenlist`, 'foo', 'polygon-manual-list' )

src/releaser/algolia/index.ts

@@ -0,0 +1,125 @@
+import algoliasearch, { SearchIndex } from "algoliasearch";
+import { networksForEach, assetsForEach } from "../../db";
+import { ObjectType, Description, ChainObject, Network, Asset } from "../../model";
+
+const updateOptions = {
+  createIfNotExists: true,
+};
+
+declare interface IndexedAsset {
+  objectID: string;
+  address?: string;
+  networkCode: string;
+  type: ObjectType;
+  name: string;
+  symbol: string;
+  description: Description | {};
+  logoUrl: string | null;
+}
+
+function transform({
+  id: objectID,
+  networkCode,
+  type,
+  name,
+  symbol,
+  description,
+  logo,
+}: ChainObject): IndexedAsset {
+  const logoParsed = logo;
+  const logoUrl = logoParsed?.svg || logoParsed?.png || null;
+  description = description || {};
+
+  return {
+    objectID,
+    networkCode,
+    type,
+    name,
+    symbol,
+    description,
+    logoUrl,
+  };
+}
+
+export async function updateAlgoliaSearch(
+  directory: string,
+  batchSize: Number = 250
+): Promise<void> {
+    if(!process.env.ALGOLIA_APP_ID) {
+        throw new Error('Missing Algolia App ID');
+    }
+
+    if(!process.env.ALGOLIA_API_KEY) {
+        throw new Error('Missing Algolia API Key');
+    }
+
+    const client = algoliasearch(process.env.ALGOLIA_APP_ID, 
+        process.env.ALGOLIA_API_KEY);
+    const assets = client.initIndex("assets");
+
+    try {
+      await assets.setSettings({
+        searchableAttributes: [
+          "objectID",
+          "unordered(name)",
+          "symbol",
+          "unordered(description)",
+          "address",
+        ],
+        attributesForFaceting: ["type", "networkCode"],
+      });
+
+      await updateNetworks(assets, batchSize, directory);
+      await updateAssets(assets, batchSize, directory);
+
+      console.log('Updated Algolia Search')
+      Promise.resolve();
+    } catch (err: any) {
+      console.error('Error with Algolia search', err);
+      throw err;
+    }
+}
+
+async function updateNetworks(index: SearchIndex, batchSize: Number, dir: string) {
+  let buffer: IndexedAsset[] = [];
+  async function updateNetwork(networkInfo: Network) {
+    if (buffer.length < batchSize) {
+      buffer.push(transform(networkInfo));
+    } else {
+      await index.partialUpdateObjects(buffer, updateOptions);
+      buffer = [];
+    }
+  }
+  async function completeUpdate() {
+    if (buffer.length > 0) {
+      const response = await index.partialUpdateObjects(buffer, updateOptions);
+    }
+  }
+
+  await networksForEach(updateNetwork, completeUpdate, dir);
+  return Promise.resolve();
+}
+
+async function updateAssets(index: SearchIndex, batchSize: Number, dir: string) {
+  let buffer: IndexedAsset[] = [];
+  async function updateAsset(assetInfo: Asset) {
+    if (buffer.length < batchSize) {
+      const asset = transform(assetInfo);
+      buffer.push({ ...asset, address: assetInfo.address });
+    } else {
+      await index.partialUpdateObjects(buffer, updateOptions);
+      buffer = [];
+    }
+
+    return Promise.resolve();
+  }
+  async function completeUpdate() {
+    if (buffer.length > 0) {
+      await index.partialUpdateObjects(buffer, updateOptions);
+    }
+    return Promise.resolve();
+  }
+
+  await assetsForEach(updateAsset, completeUpdate, dir);
+  return Promise.resolve();
+}