diff --git a/.pnp.cjs b/.pnp.cjs index 48c8a8c..9b2266d 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -38,6 +38,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@uniswap/token-lists", "npm:1.0.0-beta.30"],\ ["ava", "virtual:450599ab03d8d1251e47072c746a084399a8634df041193dd12e1f87e2f387d3ff5eceeef406d1fa24121d1005f46886286ea884cce71197cee3b5b951ae7881#npm:4.3.0"],\ ["axios", "npm:0.27.2"],\ + ["csv-parse", "npm:5.3.0"],\ ["ethereum-checksum-address", "npm:0.0.8"],\ ["jsonschema", "npm:1.4.1"],\ ["pkg", "virtual:450599ab03d8d1251e47072c746a084399a8634df041193dd12e1f87e2f387d3ff5eceeef406d1fa24121d1005f46886286ea884cce71197cee3b5b951ae7881#npm:5.7.0"],\ @@ -176,6 +177,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@uniswap/token-lists", "npm:1.0.0-beta.30"],\ ["ava", "virtual:450599ab03d8d1251e47072c746a084399a8634df041193dd12e1f87e2f387d3ff5eceeef406d1fa24121d1005f46886286ea884cce71197cee3b5b951ae7881#npm:4.3.0"],\ ["axios", "npm:0.27.2"],\ + ["csv-parse", "npm:5.3.0"],\ ["ethereum-checksum-address", "npm:0.0.8"],\ ["jsonschema", "npm:1.4.1"],\ ["pkg", "virtual:450599ab03d8d1251e47072c746a084399a8634df041193dd12e1f87e2f387d3ff5eceeef406d1fa24121d1005f46886286ea884cce71197cee3b5b951ae7881#npm:5.7.0"],\ @@ -1097,6 +1099,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["csv-parse", [\ + ["npm:5.3.0", {\ + "packageLocation": "./.yarn/cache/csv-parse-npm-5.3.0-92f0ce63f9-6754e85aef.zip/node_modules/csv-parse/",\ + "packageDependencies": [\ + ["csv-parse", "npm:5.3.0"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["currently-unhandled", [\ ["npm:0.4.1", {\ "packageLocation": "./.yarn/cache/currently-unhandled-npm-0.4.1-38eddab665-1f59fe10b5.zip/node_modules/currently-unhandled/",\ diff --git a/.yarn/cache/csv-parse-npm-5.3.0-92f0ce63f9-6754e85aef.zip b/.yarn/cache/csv-parse-npm-5.3.0-92f0ce63f9-6754e85aef.zip new file mode 100644 index 0000000..f385aef Binary files /dev/null and b/.yarn/cache/csv-parse-npm-5.3.0-92f0ce63f9-6754e85aef.zip differ diff --git a/.yarn/install-state.gz b/.yarn/install-state.gz index e8c073b..a4b1534 100644 Binary files a/.yarn/install-state.gz and b/.yarn/install-state.gz differ diff --git a/package.json b/package.json index fde01e6..397b6db 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "clean": "rimraf ./dist/ ./exec/", "build": "yarn clean && tsc", "preparePublish": "yarn build && npm version patch", - "test": "yarn build && ava --verbose", + "test": "yarn build && ava --verbose -T 5m -m \"Mappings can be read successfully\"", "bundle": "yarn build && pkg . --out-dir ./exec/" }, "devDependencies": { @@ -35,6 +35,7 @@ "dependencies": { "@uniswap/token-lists": "^1.0.0-beta.30", "axios": "^0.27.2", + "csv-parse": "^5.3.0", "ethereum-checksum-address": "^0.0.8", "jsonschema": "^1.4.1", "shelljs": "^0.8.5", diff --git a/src/db/asset.ts b/src/db/asset.ts index 8f0be6d..01ff83a 100644 --- a/src/db/asset.ts +++ b/src/db/asset.ts @@ -1,9 +1,11 @@ import { Asset } from "../model"; -import { getAssetsForNetwork, getNetworks } from "../networks"; +import { AssetMapping } from "../model/AssetMapping"; +import { getAssetMapping, getAssetsForNetwork, getNetworks } from "../networks"; import { ETHEREUM_ASSETS, POLYGON_ASSETS } from "./assets.json"; import { fetchAssetsCsv } from "./utils"; type AssetInfoCallback = (assetInfo: Asset) => Promise; +type AssetMappingInfoCallback = (assetMapping: AssetMapping) => Promise; export async function assetsForEach(callback: AssetInfoCallback, complete?: () => Promise) { try { @@ -52,6 +54,21 @@ export async function findAssetByNetworkIdAndAddress( } } +export async function assetsMappingForEach(callback: AssetMappingInfoCallback, complete?: () => Promise) { + try { + const assetMappings = await getAssetMapping(); + for (const assetMapping of assetMappings) { + await callback(assetMapping); + } + + if (complete) { + await complete(); + } + } catch (err) { + throw err; + } +} + async function getMockAssets(networkId?: string): Promise { let res = []; diff --git a/src/model/AssetMapping.ts b/src/model/AssetMapping.ts new file mode 100644 index 0000000..4d772b5 --- /dev/null +++ b/src/model/AssetMapping.ts @@ -0,0 +1,8 @@ +export class AssetMapping { + primaryId: string; // id of the asset on the primary network + primaryNetwork: string; // the primary network code (e.g. ethereum) + mapping: { + [network: string]: [string]; + }; +} + diff --git a/src/networks/index.test.ts b/src/networks/index.test.ts index 075ce7d..6d6b3ac 100644 --- a/src/networks/index.test.ts +++ b/src/networks/index.test.ts @@ -1,7 +1,14 @@ -import test from 'ava'; -import { getNetworks } from '.'; +import test from "ava"; +import { getAssetMapping, getNetworks } from "."; + +test("networks includes ethereum", async (t) => { + const networks = await getNetworks(); + t.truthy(networks.find((n) => n.name === "Ethereum")); +}); + +test("Mappings can be read successfully", async (t) => { + const mappings = await getAssetMapping(); + console.log(mappings); + t.truthy(mappings); +}); -test('networks includes ethereum', async t => { - const networks = await getNetworks(); - t.truthy(networks.find(n => n.name === 'Ethereum')); -}); \ No newline at end of file diff --git a/src/networks/index.ts b/src/networks/index.ts index 1c24213..66aeaa1 100644 --- a/src/networks/index.ts +++ b/src/networks/index.ts @@ -1,70 +1,109 @@ +import { parse } from "csv-parse/sync"; import fs from "fs"; import { Asset } from "../model"; +import { AssetMapping } from "../model/AssetMapping"; import { Network } from "../model/Network"; import { cloneOrPullRepoAndUpdateSubmodules } from "../utils"; import { DEFAULT_REPO_DISK_LOCATION, REPO_CLONE_URL } from "../utils/constants"; import { getDirectories, readAndParseJson } from "../utils/filesystem"; export async function getNetworks(dir?: string): Promise { - if(!dir) { - dir = DEFAULT_REPO_DISK_LOCATION - } + if (!dir) { + dir = DEFAULT_REPO_DISK_LOCATION; + } - const res: Network[] = []; + const res: Network[] = []; - try { + try { + await cloneOrPullRepoAndUpdateSubmodules(REPO_CLONE_URL, dir, true, "master"); - await cloneOrPullRepoAndUpdateSubmodules(REPO_CLONE_URL, dir, true, 'master'); + const directories = await getDirectories(dir); - const directories = await getDirectories(dir); + directories.forEach((directory) => { + const split = directory.split("/"); - directories.forEach(directory => { - const split = directory.split('/'); + if (split[split.length - 2] === "networks" && !directory.includes(".git")) { + res.push(readAndParseJson(`${directory}/info.json`)); + } + }); - if(split[split.length - 2] === 'networks' && !directory.includes('.git')) { - res.push(readAndParseJson(`${directory}/info.json`)); - } - }); - - if(res.length === 0) { - throw new Error(`getNetworks No networks found in ${dir}`); - } + if (res.length === 0) { + throw new Error(`getNetworks No networks found in ${dir}`); + } - return res; - } catch (err) { - throw err; - } + return res; + } catch (err) { + throw err; + } } export async function getAssetsForNetwork(network: string, dir?: string): Promise { - if(!dir) { - dir = DEFAULT_REPO_DISK_LOCATION - } - - const res: Asset[] = []; - - try { - await cloneOrPullRepoAndUpdateSubmodules(REPO_CLONE_URL, dir, true, 'master'); - - const tokenlistDir = `${dir}/networks/${network}/assets/${network}-tokenlist`; - - if(!fs.existsSync(tokenlistDir)) { - return []; - } - - // TODO, make it work for multiple tokenlists - const assetDirs = await getDirectories(tokenlistDir); - - assetDirs.forEach(directory => { - const split = directory.split('/'); - - if(split[split.length - 2] === `${network}-tokenlist` && !directory.includes('.git')) { - res.push(readAndParseJson(`${directory}/info.json`)); - } - }); - - return res; - }catch (err) { - throw err; + if (!dir) { + dir = DEFAULT_REPO_DISK_LOCATION; + } + + const res: Asset[] = []; + + try { + await cloneOrPullRepoAndUpdateSubmodules(REPO_CLONE_URL, dir, true, "master"); + + const tokenlistDir = `${dir}/networks/${network}/assets/${network}-tokenlist`; + + if (!fs.existsSync(tokenlistDir)) { + return []; } -} \ No newline at end of file + + // TODO, make it work for multiple tokenlists + const assetDirs = await getDirectories(tokenlistDir); + + assetDirs.forEach((directory) => { + const split = directory.split("/"); + + if (split[split.length - 2] === `${network}-tokenlist` && !directory.includes(".git")) { + res.push(readAndParseJson(`${directory}/info.json`)); + } + }); + + return res; + } catch (err) { + throw err; + } +} + +export async function getAssetMapping(dir: string = DEFAULT_REPO_DISK_LOCATION): Promise { + const parseRecord = (networks, record) => { + const assetMapping = new AssetMapping(); + assetMapping.primaryId = record.primaryId; + assetMapping.primaryNetwork = record.primaryNetwork; + assetMapping.mapping = {}; + + networks + .filter((network) => network != record.primaryNetwork) + .filter((network) => record[network].length > 0) + .forEach((network) => { + const identifiers = record[network].split(";"); + assetMapping.mapping[network] = identifiers; + }); + + return assetMapping; + }; + + const filename = `${dir}/assets.tsv`; + const data = fs.readFileSync(filename); + const records = parse(data, { + delimiter: "\t", + columns: true, + skip_empty_lines: true, + }); + + const networks = Object.keys(records[0]).filter( + (key) => !["primaryId", "primaryNetwork", "name", "symbol"].includes(key) + ); + + return Promise.resolve( + records + .map((record) => parseRecord(networks, record)) + .filter((assetMapping) => Object.keys(assetMapping.mapping).length > 0) + ); +} + diff --git a/yarn.lock b/yarn.lock index 0f04416..ee75de4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -131,6 +131,7 @@ __metadata: "@uniswap/token-lists": ^1.0.0-beta.30 ava: ^4.3.0 axios: ^0.27.2 + csv-parse: ^5.3.0 ethereum-checksum-address: ^0.0.8 jsonschema: ^1.4.1 pkg: ^5.7.0 @@ -930,6 +931,13 @@ __metadata: languageName: node linkType: hard +"csv-parse@npm:^5.3.0": + version: 5.3.0 + resolution: "csv-parse@npm:5.3.0" + checksum: 6754e85aef9dfdf19da99206b4a83f51fb881b3a75f9f7b6221864252c386410e1f535a013911d6e52f77a017d3dadf10f1e526fe31e01b2b79074c3fb4b3aaa + languageName: node + linkType: hard + "currently-unhandled@npm:^0.4.1": version: 0.4.1 resolution: "currently-unhandled@npm:0.4.1"