diff --git a/src/index.ts b/src/index.ts index b0259bc..5b8f048 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,3 @@ -import { RepoFileGenerator } from './tsv/RepoFileGenerator'; import { DEFAULT_TEMP_DIR } from './utils/constants'; import { validate } from './validate'; @@ -10,7 +9,6 @@ export * from './trustwallet'; export * from './utils'; export * from './validate'; export * from './db'; -export * from './tsv'; // const base = '/Users/ap/ama_dev/map3/indexer/tmp'; // ingestTokenList(`${base}/kq.tokenlist.json`, `${base}/map3xyz-assets/networks/polygon/assets/polygon-tokenlist`, 'foo', 'polygon-manual-list' ) @@ -27,7 +25,3 @@ export * from './tsv'; // validate('all', '/Users/ap/ama_dev/smartchain-tokenlist/smartchain-tokenlist') // .then(console.log) // .catch(console.error); - -// RepoFileGenerator.generate() -// .then(csv => csv.deserialise(DEFAULT_TEMP_DIR)) -// .catch(console.error); diff --git a/src/tsv/AssetsTsv.ts b/src/tsv/AssetsTsv.ts deleted file mode 100644 index af02d05..0000000 --- a/src/tsv/AssetsTsv.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { AssetsTsvRow } from "./AssetsTsvRow"; -import { RepoPointer } from "./types"; -import fs from 'fs'; -import path from 'path'; -import { getNetworks } from "../networks"; - -const HEADER_ROW_START = 'primaryId\tprimaryNetwork\tname\tsymbol'; - -interface IAssetsTsv { - rows: AssetsTsvRow[]; - append: (row: AssetsTsvRow) => void; - remove: (primaryId: RepoPointer) => void; -} - -export class AssetsTsv implements IAssetsTsv { - rows: AssetsTsvRow[] = []; - - append (row: AssetsTsvRow): AssetsTsvRow { - // TODO; make this more defensive to not insert if name and symbol match - - if(!row || !row.primaryId) { - throw new Error('AssetsTsv.append: row has no primaryId ' + JSON.stringify(row)); - } - - const existing = this.get(row.primaryId); - if(existing) { - throw new Error('AssetsTsv.append: row already exists. Existing: ' + JSON.stringify(existing) + ' New: ' + JSON.stringify(row)); - } - - this.rows.push(row); - return row; - }; - - remove(primaryId: RepoPointer): AssetsTsvRow { - let index; - const row = this.rows.find((_row, i) => { - if(_row.primaryId.toLowerCase() === primaryId.toLowerCase()) { - index = i; - return true; - } - return false; - }); - - this.rows = this.rows.splice(index, 1); - return row; - } - - replace(row: AssetsTsvRow): AssetsTsvRow { - this.rows = this.rows.map(_row => { - return _row.primaryId === row.primaryId? row : _row; - }); - return row; - } - - get(primaryId: RepoPointer): AssetsTsvRow { - if(this.rows.find(row => !row || row.primaryId === undefined)) { - throw new Error('found row without primary key '); - } - return this.rows.find((row, i) => { - return row.primaryId.toLowerCase() === primaryId.toLowerCase() - }); - }; - - assetExistsWithNameOrSymbol(name: string, symbol: string): boolean { - return this.rows.find(row => row.name.toLowerCase() === name.toLowerCase() - || row.symbol.toLowerCase() === symbol.toLowerCase()) !== undefined; - } - - static async parse(tsvLocation: string): Promise { - try { - const tsv = await fs.promises.readFile(tsvLocation, 'utf8'); - const rows = tsv.split('\n').map(row => row.split('\t')); - const headers = rows[0]; - const assetsTsv = new AssetsTsv(); - - for (const row in rows.slice(1)) { - const primaryId = row[0] as RepoPointer; - const name = row[2] as string; - const symbol = row[3] as string; - const primaryNetwork = row[1]; - const networks = {}; - - headers.slice(2).forEach((header, i) => { - networks[header] = row[i + 2].split(';'); - } - ); - - assetsTsv.append(await AssetsTsvRow.prepare(primaryId, primaryNetwork, name, symbol, networks)); - } - - return assetsTsv; - } catch (err) { - throw err; - } - }; - - async deserialise(persistDir: string, fileName: string = 'assets.tsv'): Promise { - try { - if(!fs.existsSync(persistDir)) { - throw `AssetsTsv.deserialise: persistDir ${persistDir} does not exist`; - } - - const networkDirs = (await getNetworks()).map(network => network.networkCode); - - let tsv = `${HEADER_ROW_START}\t${networkDirs.sort().join('\t')}\n`; - - this.rows.forEach(row => { - row.cleanIds(); - tsv += `${row.primaryId}\t${row.primaryNetwork}\t${row.name}\t${row.symbol}\t${networkDirs.sort().map(network => row.networks[network].join(';')).join('\t')}\n`; - }); - - return fs.promises.writeFile(path.join(persistDir, fileName), tsv); - } catch (err) { - throw err; - } - }; -} \ No newline at end of file diff --git a/src/tsv/AssetsTsvRow.ts b/src/tsv/AssetsTsvRow.ts deleted file mode 100644 index 78e1426..0000000 --- a/src/tsv/AssetsTsvRow.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { getNetworks } from "../networks"; -import { RepoPointer } from "./types"; - -let networkDirs = []; - -interface IAssetsTsvRow { - primaryId: RepoPointer; - primaryNetwork: string; - name: string; - symbol: string; - networks: { - [network: string]: RepoPointer[] - }; -} - -export class AssetsTsvRow implements IAssetsTsvRow { - primaryId: RepoPointer; - primaryNetwork: string; - name: string; - symbol: string; - networks: { [network: string]: RepoPointer[]; }; - - private constructor(primaryId: RepoPointer, primaryNetwork: string, name: string, symbol: string, networks: { [network: string]: RepoPointer[]; }) { - if(!primaryId.startsWith('id:')) { - throw new Error(`AssetsTsvRow primaryId ${primaryId} must start with 'id:'`); - } - - this.primaryId = primaryId; - this.primaryNetwork = primaryNetwork.toLowerCase(); - - if(!networks[primaryNetwork] || !networks[primaryNetwork].includes(primaryId)) { - throw new Error(`AssetsTsvRow primaryId ${primaryId} must be in network column ${primaryNetwork}`); - } - - this.networks = networks; - this.name = name; - this.symbol = symbol.toUpperCase(); - } - - cleanIds() { - this.primaryId = this.primaryId.split("id:")[1]; - Object.keys(this.networks).forEach(network => { - this.networks[network] = this.networks[network].map(id => id.split("id:")[1]); - }); - } - - static async prepare(primaryId: RepoPointer, primaryNetwork: string, name: string, symbol: string, networks: { [network: string]: RepoPointer[];}): Promise { - try { - if(networkDirs.length === 0) { - networkDirs = (await getNetworks()).map(network => network.networkCode); - } - - if(!networkDirs.includes(primaryNetwork)) { - throw new Error(`AssetsTsvRow primaryNetwork ${primaryNetwork} must be a valid network`); - } - - Object.keys(networks).forEach(n => { - if(!networkDirs.includes(n)) { - throw new Error(`AssetsTsvRow network ${n} in networks list must be a valid network`); - } - }); - - if(!networks[primaryNetwork].includes(primaryId)) { - networks[primaryNetwork].push(primaryId); - } - - return new AssetsTsvRow(primaryId, primaryNetwork, name, symbol, networks); - } catch (err) { - throw err; - } - } -} \ No newline at end of file diff --git a/src/tsv/RepoFileGenerator.test.ts b/src/tsv/RepoFileGenerator.test.ts deleted file mode 100644 index 7a8a954..0000000 --- a/src/tsv/RepoFileGenerator.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import test from 'ava'; -import { DEFAULT_TEMP_DIR } from '../utils/constants'; -import { RepoFileGenerator } from './RepoFileGenerator'; - -test('Generating and saving csv to tmp folder works ', async t => { - try { - // const csv = await RepoFileGenerator.generate(); - // csv.deserialise(DEFAULT_TEMP_DIR + '/test-assets.csv'); - t.pass(); - } catch (err) { - t.fail(err.message); - } -}); \ No newline at end of file diff --git a/src/tsv/RepoFileGenerator.ts b/src/tsv/RepoFileGenerator.ts deleted file mode 100644 index 0c2ea1f..0000000 --- a/src/tsv/RepoFileGenerator.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { Asset } from "../model"; -import { getAssetsForNetwork, getNetworks } from "../networks"; -import { shallowClone } from "../utils"; -import { DEFAULT_REPO_DISK_LOCATION } from "../utils/constants"; -import { AssetsTsv, AssetsTsvRow } from "."; -import { EXAMPLE_ASSET_MAP } from "./tmp-maps.json"; - -/* - For every network and asset within each network in the repo, - it check if the network asset is mapped to another one. If so it adds it, - else it checks if there's already an asset with the same name or symbol. If so skips it - else finally it adds the asset or network asset to the CSV -*/ -export class RepoFileGenerator { - - static async generate(repoLoc?: string): Promise { - - const assetsCsv = new AssetsTsv(); - - if(!repoLoc) { - repoLoc = DEFAULT_REPO_DISK_LOCATION - } - - try { - const networks = await getNetworks(repoLoc); - let networksMap = {}; - networks.map(n => n.networkCode).forEach(networkCode => { - networksMap[networkCode] = []; - }); - - // TODO; add support for testnets fetching - for (const network of networks) { - - const mainNetworkAssetMappedTo = EXAMPLE_ASSET_MAP.find(a => a.fromNetwork === network.networkCode && a.fromAsset === `id:${network.id}`); - - let row = assetsCsv.get(`id:${network.id}`); - - if(mainNetworkAssetMappedTo) { - if(row) { - // delete the row and replace it with the mapped one - row = assetsCsv.remove(`id:${network.id}`); - row = assetsCsv.get(mainNetworkAssetMappedTo.toAsset) - } else { - - row = assetsCsv.get(mainNetworkAssetMappedTo.toAsset); - if(!row) { - // handle case where the asset its mapped to does not exist, yet. - const assetInfo = networks.find(n => n.networkCode === mainNetworkAssetMappedTo.toAsset.split(':')[1]) - || (await getAssetsForNetwork(mainNetworkAssetMappedTo.toNetwork, repoLoc)).find(a => a.id === mainNetworkAssetMappedTo.toAsset.split(":")[1]); - row = assetsCsv.append(await AssetsTsvRow.prepare(`id:${assetInfo.id}`, assetInfo.networkCode, assetInfo.name, assetInfo.symbol, shallowClone(networksMap))); - } - } - row.networks[mainNetworkAssetMappedTo.fromNetwork].push(mainNetworkAssetMappedTo.fromAsset); - row = assetsCsv.replace(row); - } else if (!row) { - - // ignore and log if there is one with the same symbol or name? - if(assetsCsv.assetExistsWithNameOrSymbol(network.name, network.symbol)) { - console.log(`Skipping network ${network.id} because it has the same name or symbol as another asset`); - } else { - // create the network asset - assetsCsv.append(await AssetsTsvRow.prepare(`id:${network.id}`, network.networkCode, network.name, network.symbol, shallowClone(networksMap))); - } - } - - const networkAssets: Asset[] = await getAssetsForNetwork(network.networkCode, repoLoc); - - if(!networkAssets || networkAssets.length === 0) { - continue; - } - - for(const asset of networkAssets) { - const assetMappedToAnotherOne = EXAMPLE_ASSET_MAP.find(a => a.fromNetwork === network.networkCode && a.fromAsset === `id:${asset.id}`); - let row = assetsCsv.get(`id:${asset.id}`); - - if(assetMappedToAnotherOne) { - if(row) { - // delete the row and replace it with the mapped one - row = assetsCsv.remove(`id:${asset.id}`); - row = assetsCsv.get(assetMappedToAnotherOne.toAsset) - } else { - row = assetsCsv.get(assetMappedToAnotherOne.toAsset); - - if(!row) { - // handle case where the asset its mapped to does not exist, yet. - const assetInfo = networks.find(n => n.networkCode === mainNetworkAssetMappedTo.toAsset.split(':')[1]) - || (await getAssetsForNetwork(assetMappedToAnotherOne.toNetwork, repoLoc)).find(a => a.id === assetMappedToAnotherOne.toAsset.split(":")[1]); - row = assetsCsv.append(await AssetsTsvRow.prepare(`id:${assetInfo.id}`, assetInfo.networkCode, assetInfo.name, assetInfo.symbol, shallowClone(networksMap))); - } - } - row.networks[assetMappedToAnotherOne.fromNetwork].push(assetMappedToAnotherOne.fromAsset); - row = assetsCsv.replace(row); - } else if (!row) { - - // ignore and log if there is one with the same symbol or name? - if(assetsCsv.assetExistsWithNameOrSymbol(asset.name, asset.symbol)) { - console.log(`Skipping asset ${asset.id} because it has the same name or symbol as another asset`); - continue; - } - // create the network asset - assetsCsv.append(await AssetsTsvRow.prepare(`id:${asset.id}`, asset.networkCode, asset.name, asset.symbol, shallowClone(networksMap))); - } - } - } - - return assetsCsv; - - } catch (err) { - throw err; - } - } -} \ No newline at end of file diff --git a/src/tsv/index.ts b/src/tsv/index.ts deleted file mode 100644 index 5cca23f..0000000 --- a/src/tsv/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './AssetsTsv'; -export * from './AssetsTsvRow'; -export * from './types'; -export * from './RepoFileGenerator'; \ No newline at end of file diff --git a/src/tsv/tmp-maps.json.ts b/src/tsv/tmp-maps.json.ts deleted file mode 100644 index dd6e38e..0000000 --- a/src/tsv/tmp-maps.json.ts +++ /dev/null @@ -1,41 +0,0 @@ -export const EXAMPLE_ASSET_MAP = [ - // { - // // matic on the polygon network to matic on the ethereum network - // "fromAsset": "id:d6bffe69-071d-4fa8-9038-d90fac19bf77", - // 'fromNetwork': 'polygon', - // "toAsset": "id:287bd359-17a3-4bba-9e20-a1569f955452", - // 'toNetwork': 'ethereum', - // "type": "contract", - // "verifications": [ - // { - // "type" : "kleros", - // "submission": "0x163ec6ab7788eba525a29d1e91fce264c2e3fa7d7b1a25ecc874b3d80e677809" - // } - // ] - // }, - { - // USDC on polygon to USDC on ETH - "fromAsset": "id:fe2bf2f8-3ddc-4ccc-8f34-8fdd9be03884", - 'fromNetwork': 'polygon', - "toAsset": "id:53adbb94-6a68-4eeb-af49-6b6d9e84a1f4", - 'toNetwork': 'ethereum', - "type": "direct_issuance", - "verifications": [ - { - "type" : "kleros", - "submission": "0x163ec6ab7788eba525a29d1e91fce264c2e3fa7d7b1a25ecc874b3d80e677809" - } - ] - } - - // // // USDC on polygon to USDC on ETH based on addresses - - // { - // "fromAsset": "0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174", - // "fromNetwork": "polygon", - // "toAsset": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", - // "toNetwork": "ethereum", - // "type": "direct_issuance", - // "verifications": [] - // } -]; \ No newline at end of file diff --git a/src/tsv/types.ts b/src/tsv/types.ts deleted file mode 100644 index e3477ee..0000000 --- a/src/tsv/types.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Asset id field, uuid of the asset. -export type RepoPointer = `${string}`; \ No newline at end of file