Added support transactions and other links

rusregistry/entities/other_connection.py

@@ -0,0 +1,18 @@
+from entities.entity import Entity
+import pandas as pd
+
+class OtherConnection(Entity):
+    def __init__(self, item):
+        super().__init__(item)
+        self.entity_type = 'UnknownLink'
+
+    def from_csv_row(self, row: pd.core.series.Series):
+        description = f"Type: {row['type']}, Form: {row['form']}, Sum: {row['s']}"
+        self.data_dict = {'object': row['inn'],
+                          'subject': row['from_inn'],
+                          'date': row['accept_date'],
+                          'description': description}
+
+    def make_id(self, entity):
+        entity.id = self.data_dict['object'] + 'support' + self.data_dict['subject']
+        return entity

rusregistry/entities/transactions.py

@@ -0,0 +1,19 @@
+from entities.entity import Entity
+import pandas as pd
+
+class SupportTransaction(Entity):
+    def __init__(self, item):
+        super().__init__(item)
+        self.entity_type = 'Payment'
+
+    def from_csv_row(self, row: pd.core.series.Series):
+        description = f"Type: {row['type']}, Form: {row['form']}"
+        self.data_dict = {'beneficiary': row['inn'],
+                          'payer': row['from_inn'],
+                          'date': row['accept_date'],
+                          'amount': row['s'],
+                          'description': description}
+
+    def make_id(self, entity):
+        entity.id = self.data_dict['payer'] + 'transaction' + self.data_dict['beneficiary']
+        return entity

rusregistry/transactions-run.ipynb

@@ -0,0 +1,184 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "66318a44-ffc6-4ed9-8189-9d5a8c0dd9f3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from entities.other_connection import OtherConnection\n",
+    "from entities.transactions import SupportTransaction\n",
+    "import followthemoney as ftm\n",
+    "import pandas as pd\n",
+    "from csv_processing.utils import add_id_prefix"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "580a0e63-44de-4240-bcdb-864e2905476f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_csv('../data1/support2021.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "bfed5cfa-56d0-4438-8b84-608d9c41f374",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ref_type = pd.read_csv('../data1/support_type_ref.csv')\n",
+    "ref_form = pd.read_csv('../data1/support_form_ref.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "b0f0450a-7312-4686-a3a3-546937948b72",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Loading support forms and types description (translated by google translate!)\n",
+    "support_forms = ref_form.set_index('id').to_dict()['english_version']\n",
+    "support_types = ref_type.set_index('id').to_dict()['english_version']\n",
+    "\n",
+    "df['form'] = df['form_id'].apply(support_forms.get)\n",
+    "df['type'] = df['type_id'].apply(support_types.get)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "2fd429b3-ffa7-4460-b87a-708d854dda5b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Financial support      3796037\n",
+       "Consulting support     1516093\n",
+       "Educational support     493066\n",
+       "Information support     211449\n",
+       "Property support         54223\n",
+       "Innovative support        5547\n",
+       "Name: form, dtype: int64"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df['form'].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "1d37ab8f-aa52-4532-b6be-7e039a142e84",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Support docs has only INN-keys, \n",
+    "# so we need to check if INN is for a company,\n",
+    "# and we can use our primary key OGRN \n",
+    "\n",
+    "companies = pd.concat([pd.read_csv('../data1/org_2022-04-27_08:31:04.csv'),\n",
+    "                       pd.read_csv('../data1/org_2022-03-17_20:33:51.csv')])\n",
+    "inn2ogrn = companies.set_index('inn').to_dict()['ogrn']\n",
+    "\n",
+    "\n",
+    "def make_id(inn):\n",
+    "    ogrn = inn2ogrn.get(inn)\n",
+    "    if ogrn is None:\n",
+    "        return add_id_prefix(fix_inn(inn), 'inn')\n",
+    "    return add_id_prefix(ogrn, 'ogrn')\n",
+    "\n",
+    "df['inn'] = df['inn'].apply(make_id)\n",
+    "df['from_inn'] = df['from_inn'].apply(make_id)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "75f7eeee-5760-4500-88a2-150f5c782a0e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Separating financial support from other types\n",
+    "financial_support = df[df['form_id'] == 100]\n",
+    "other_support = df[df['form_id'] != 100]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "1d001059-9f6f-45bd-b21d-cb3d82b5726d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|█████████████████████████████████████████████████████████████████████| 3796037/3796037 [10:12<00:00, 6201.77it/s]\n",
+      "100%|█████████████████████████████████████████████████████████████████████| 2280378/2280378 [05:51<00:00, 6482.47it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from tqdm import tqdm\n",
+    "data = []\n",
+    "for i, row in tqdm(financial_support.iterrows(), total=len(financial_support)):\n",
+    "    data.append(SupportTransaction(row).to_ftm().to_dict())\n",
+    "for i, row in tqdm(other_support.iterrows(), total=len(other_support)):\n",
+    "    data.append(OtherConnection(row).to_ftm().to_dict())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "4f77e1ba-a580-4696-9013-8322d17bc0f2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import gzip\n",
+    "\n",
+    "jsonfilename = 'output/ru-egrul-transactions.json.gzip'\n",
+    "\n",
+    "# WRITE\n",
+    "json_str = json.dumps(data) + \"\\n\"               \n",
+    "json_bytes = json_str.encode('utf-8')   \n",
+    "\n",
+    "with gzip.open(jsonfilename, 'w') as fout:    \n",
+    "    fout.write(json_bytes) "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}