Merge pull request #28 from map3xyz/arshadm/get-mapping

Added support for reading asset mappings

package.json

@@ -35,6 +35,7 @@
   "dependencies": {
     "@uniswap/token-lists": "^1.0.0-beta.30",
     "axios": "^0.27.2",
+    "csv-parse": "^5.3.0",
     "ethereum-checksum-address": "^0.0.8",
     "jsonschema": "^1.4.1",
     "shelljs": "^0.8.5",

src/db/asset.ts

@@ -1,9 +1,11 @@
 import { Asset } from "../model";
-import { getAssetsForNetwork, getNetworks } from "../networks";
+import { AssetMapping } from "../model/AssetMapping";
+import { getAssetMapping, getAssetsForNetwork, getNetworks } from "../networks";
 import { ETHEREUM_ASSETS, POLYGON_ASSETS } from "./assets.json";
 import { fetchAssetsCsv } from "./utils";
 
 type AssetInfoCallback = (assetInfo: Asset) => Promise<void>;
+type AssetMappingInfoCallback = (assetMapping: AssetMapping) => Promise<void>;
 
 export async function assetsForEach(callback: AssetInfoCallback, complete?: () => Promise<void>) {
   try {
@@ -52,6 +54,21 @@ export async function findAssetByNetworkIdAndAddress(
   }
 }
 
+export async function assetsMappingForEach(callback: AssetMappingInfoCallback, complete?: () => Promise<void>) {
+  try {
+    const assetMappings = await getAssetMapping();
+    for (const assetMapping of assetMappings) {
+      await callback(assetMapping);
+    }
+
+    if (complete) {
+      await complete();
+    }
+  } catch (err) {
+    throw err;
+  }
+}
+
 async function getMockAssets(networkId?: string): Promise<Asset[]> {
   let res = [];
 

src/model/AssetMapping.ts

@@ -0,0 +1,8 @@
+export class AssetMapping {
+  primaryId: string; // id of the asset on the primary network
+  primaryNetwork: string; // the primary network code (e.g. ethereum)
+  mapping: {
+    [network: string]: [string];
+  };
+}
+

src/networks/index.test.ts

@@ -1,7 +1,13 @@
-import test from 'ava';
-import { getNetworks } from '.';
+import test from "ava";
+import { getAssetMapping, getNetworks } from ".";
+
+test("networks includes ethereum", async (t) => {
+  const networks = await getNetworks();
+  t.truthy(networks.find((n) => n.name === "Ethereum"));
+});
+
+test("Mappings can be read successfully", async (t) => {
+  const mappings = await getAssetMapping();
+  t.truthy(mappings.length > 0);
+});
 
-test('networks includes ethereum', async t => {
-    const networks = await getNetworks();
-    t.truthy(networks.find(n => n.name === 'Ethereum'));
-});

src/networks/index.ts

@@ -1,70 +1,109 @@
+import { parse } from "csv-parse/sync";
 import fs from "fs";
 import { Asset } from "../model";
+import { AssetMapping } from "../model/AssetMapping";
 import { Network } from "../model/Network";
 import { cloneOrPullRepoAndUpdateSubmodules } from "../utils";
 import { DEFAULT_REPO_DISK_LOCATION, REPO_CLONE_URL } from "../utils/constants";
 import { getDirectories, readAndParseJson } from "../utils/filesystem";
 
 export async function getNetworks(dir?: string): Promise<Network[]> {
-   if(!dir)  {
-    dir = DEFAULT_REPO_DISK_LOCATION
-   }
+  if (!dir) {
+    dir = DEFAULT_REPO_DISK_LOCATION;
+  }
 
-   const res: Network[] = [];
+  const res: Network[] = [];
 
-   try {
+  try {
+    await cloneOrPullRepoAndUpdateSubmodules(REPO_CLONE_URL, dir, true, "master");
 
-        await cloneOrPullRepoAndUpdateSubmodules(REPO_CLONE_URL, dir, true, 'master');
+    const directories = await getDirectories(dir);
 
-        const directories = await getDirectories(dir);
+    directories.forEach((directory) => {
+      const split = directory.split("/");
 
-        directories.forEach(directory => {
-            const split = directory.split('/');
+      if (split[split.length - 2] === "networks" && !directory.includes(".git")) {
+        res.push(readAndParseJson(`${directory}/info.json`));
+      }
+    });
 
-            if(split[split.length - 2] === 'networks' && !directory.includes('.git')) {
-                res.push(readAndParseJson(`${directory}/info.json`));
-            }
-        });
-
-        if(res.length === 0) {
-            throw new Error(`getNetworks No networks found in ${dir}`);
-        }
+    if (res.length === 0) {
+      throw new Error(`getNetworks No networks found in ${dir}`);
+    }
 
-        return res;
-   } catch (err) {  
-        throw err;
-   }
+    return res;
+  } catch (err) {
+    throw err;
+  }
 }
 
 export async function getAssetsForNetwork(network: string, dir?: string): Promise<Asset[]> {
-    if(!dir)  {
-        dir = DEFAULT_REPO_DISK_LOCATION
-       }
-
-       const res: Asset[] = [];
-
-       try {
-        await cloneOrPullRepoAndUpdateSubmodules(REPO_CLONE_URL, dir, true, 'master');
-
-        const tokenlistDir = `${dir}/networks/${network}/assets/${network}-tokenlist`;
-
-        if(!fs.existsSync(tokenlistDir)) {
-            return [];
-        }
-
-        // TODO, make it work for multiple tokenlists
-        const assetDirs = await getDirectories(tokenlistDir);
-
-        assetDirs.forEach(directory => {
-            const split = directory.split('/');
-
-            if(split[split.length - 2] === `${network}-tokenlist` && !directory.includes('.git')) {
-                res.push(readAndParseJson(`${directory}/info.json`));
-            }
-        });
-
-        return res;
-    }catch (err) {
-        throw err;
+  if (!dir) {
+    dir = DEFAULT_REPO_DISK_LOCATION;
+  }
+
+  const res: Asset[] = [];
+
+  try {
+    await cloneOrPullRepoAndUpdateSubmodules(REPO_CLONE_URL, dir, true, "master");
+
+    const tokenlistDir = `${dir}/networks/${network}/assets/${network}-tokenlist`;
+
+    if (!fs.existsSync(tokenlistDir)) {
+      return [];
     }
-}
+
+    // TODO, make it work for multiple tokenlists
+    const assetDirs = await getDirectories(tokenlistDir);
+
+    assetDirs.forEach((directory) => {
+      const split = directory.split("/");
+
+      if (split[split.length - 2] === `${network}-tokenlist` && !directory.includes(".git")) {
+        res.push(readAndParseJson(`${directory}/info.json`));
+      }
+    });
+
+    return res;
+  } catch (err) {
+    throw err;
+  }
+}
+
+export async function getAssetMapping(dir: string = DEFAULT_REPO_DISK_LOCATION): Promise<AssetMapping[]> {
+  const parseRecord = (networks, record) => {
+    const assetMapping = new AssetMapping();
+    assetMapping.primaryId = record.primaryId;
+    assetMapping.primaryNetwork = record.primaryNetwork;
+    assetMapping.mapping = {};
+
+    networks
+      .filter((network) => network != record.primaryNetwork)
+      .filter((network) => record[network].length > 0)
+      .forEach((network) => {
+        const identifiers = record[network].split(";");
+        assetMapping.mapping[network] = identifiers;
+      });
+
+    return assetMapping;
+  };
+
+  const filename = `${dir}/assets.tsv`;
+  const data = fs.readFileSync(filename);
+  const records = parse(data, {
+    delimiter: "\t",
+    columns: true,
+    skip_empty_lines: true,
+  });
+
+  const networks = Object.keys(records[0]).filter(
+    (key) => !["primaryId", "primaryNetwork", "name", "symbol"].includes(key)
+  );
+
+  return Promise.resolve(
+    records
+      .map((record) => parseRecord(networks, record))
+      .filter((assetMapping) => Object.keys(assetMapping.mapping).length > 0)
+  );
+}
+

yarn.lock

@@ -131,6 +131,7 @@ __metadata:
     "@uniswap/token-lists": ^1.0.0-beta.30
     ava: ^4.3.0
     axios: ^0.27.2
+    csv-parse: ^5.3.0
     ethereum-checksum-address: ^0.0.8
     jsonschema: ^1.4.1
     pkg: ^5.7.0
@@ -930,6 +931,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"csv-parse@npm:^5.3.0":
+  version: 5.3.0
+  resolution: "csv-parse@npm:5.3.0"
+  checksum: 6754e85aef9dfdf19da99206b4a83f51fb881b3a75f9f7b6221864252c386410e1f535a013911d6e52f77a017d3dadf10f1e526fe31e01b2b79074c3fb4b3aaa
+  languageName: node
+  linkType: hard
+
 "currently-unhandled@npm:^0.4.1":
   version: 0.4.1
   resolution: "currently-unhandled@npm:0.4.1"