From 33f23e05e152e14b4b4709c17f7c1c4c994bd2a6 Mon Sep 17 00:00:00 2001 From: raynardj Date: Sat, 30 Apr 2022 18:25:24 +0800 Subject: [PATCH] cleaning nbs --- nbs/07_freemap.ipynb | 318 ---------------------------------- nbs/33_files.ipynb | 242 ++++---------------------- nbs/51_image_widgets.ipynb | 165 +----------------- nbs/CUDA_GPU_Management.ipynb | 158 ++--------------- 4 files changed, 58 insertions(+), 825 deletions(-) delete mode 100644 nbs/07_freemap.ipynb diff --git a/nbs/07_freemap.ipynb b/nbs/07_freemap.ipynb deleted file mode 100644 index 4efb280..0000000 --- a/nbs/07_freemap.ipynb +++ /dev/null @@ -1,318 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# FreeMap\n", - "> ```map``` function on free json like structure" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp freemap" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from collections import OrderedDict" - ] - }, - { - "cell_type": "code", - "execution_count": 123, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "class FreeMap:\n", - " def __init__(self,\n", - " func= lambda x:x,\n", - " filter_function = lambda x: True,\n", - " flatten = False):\n", - " \"\"\"\n", - " run map operation on a free stucture, \n", - " like dictionary, list within dictionary, etc\n", - " \n", - " \"\"\"\n", - " self.func = func\n", - " self.filter_function = filter_function\n", - " self.flatten = flatten\n", - " if flatten:\n", - " self.result_list = []\n", - " self.create_doc()\n", - " \n", - " def __repr__(self):\n", - " return f\"\"\"\n", - " Please put in one of the following type: \n", - " dict, list, tuple, set, OrderedDict\n", - " \n", - " {self.keep_structure}\n", - " ======\n", - " Mapped function\n", - " {self.func}\n", - " Value Filter function\n", - " {self.filter_function}\n", - " \"\"\"\n", - " \n", - " @property\n", - " def keep_structure(self):\n", - " return \"This operation will flatten the structure to a list\"\\\n", - " if self.flatten else\\\n", - " \"This operation will keep the original structure\"\n", - " \n", - " def create_doc(self):\n", - " doc = f\"\"\"\n", - " map function for list,dict,tuple,set,OrderedDict\n", - " {self.keep_structure}\n", - " \n", - " \"\"\"\n", - " if hasattr(self.func,\"__doc__\"):\n", - " if self.func.__doc__!=None:\n", - " doc += f\"doc string from mapping function:\\n\\t\\t{self.func.__doc__}\\t\"\n", - " setattr(self,\"__doc__\",doc)\n", - " \n", - " def parse(self,thing):\n", - " if type(thing) in [list,tuple,set]:\n", - " return self.parse_seq(thing)\n", - " elif type(thing) in [dict,OrderedDict]:\n", - " return self.parse_kv(thing)\n", - " else:\n", - " return self.parse_value(thing)\n", - " \n", - " def parse_seq(self,thing):\n", - " return type(thing)(self.parse(i) for i in thing)\n", - " \n", - " def parse_kv(self,thing):\n", - " return type(thing)((k,self.parse(v)) for k,v in thing.items())\n", - " \n", - " def parse_value(self,thing):\n", - " if self.filter_function(thing):\n", - " if self.flatten==False:\n", - " return self.func(thing)\n", - " else:\n", - " self.result_list.append(self.func(thing))\n", - " return thing\n", - " \n", - " def __call__(self,iterable):\n", - " result = self.parse(iterable)\n", - " if self.flatten:\n", - " return self.result_list\n", - " else:\n", - " return result" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Examples" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [], - "source": [ - "from math import floor" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [], - "source": [ - "examp = {\"somekey\":\"somestring\",\n", - " \"somekey_with_heavy_value\":[1,\n", - " \"2\",\n", - " \"word\",\n", - " {\"function\":floor},\n", - " {\"are\",\n", - " \"is\",\n", - " \"is\",\n", - " (\"Lovely\",\n", - " \"example\",)}],\n", - " \"someanother_key\":\"hello this is a long string with verbosity\"}" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\n", - " Please put in one of the following type: \n", - " dict, list, tuple, set, OrderedDict\n", - " \n", - " This operation will keep the original structure\n", - " ======\n", - " Mapped function\n", - " at 0x10fc0fe60>\n", - " Value Filter function\n", - " at 0x10fc0fef0>\n", - " " - ] - }, - "execution_count": 102, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "capitalize = FreeMap(lambda x:x.upper(),lambda x:(type(x)==str))\n", - "capitalize" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'somekey': 'SOMESTRING',\n", - " 'somekey_with_heavy_value': [1,\n", - " '2',\n", - " 'WORD',\n", - " {'function': },\n", - " {('LOVELY', 'EXAMPLE'), 'ARE', 'IS'}],\n", - " 'someanother_key': 'HELLO THIS IS A LONG STRING WITH VERBOSITY'}" - ] - }, - "execution_count": 103, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "capitalize(examp)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Flatten and map" - ] - }, - { - "cell_type": "code", - "execution_count": 121, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['SOMESTRING',\n", - " '2',\n", - " 'WORD',\n", - " 'IS',\n", - " 'LOVELY',\n", - " 'EXAMPLE',\n", - " 'ARE',\n", - " 'HELLO THIS IS A LONG STRING WITH VERBOSITY']" - ] - }, - "execution_count": 121, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "FreeMap(lambda x:x.upper(),lambda x:(type(x)==str),flatten=True)(examp)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Just simple flatten, the default map function doesn't do anything" - ] - }, - { - "cell_type": "code", - "execution_count": 124, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['somestring',\n", - " 1,\n", - " '2',\n", - " 'word',\n", - " ,\n", - " 'is',\n", - " 'Lovely',\n", - " 'example',\n", - " 'are',\n", - " 'hello this is a long string with verbosity']" - ] - }, - "execution_count": 124, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "FreeMap(flatten=True)(examp)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/nbs/33_files.ipynb b/nbs/33_files.ipynb index 9110fb5..57a87b0 100644 --- a/nbs/33_files.ipynb +++ b/nbs/33_files.ipynb @@ -10,100 +10,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "# default_exp files" + "from forgebox.files import get_files, file_detail" ] }, { "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "from typing import Union, List\n", - "from forgebox.imports import Path, pd\n", - "\n", - "def get_files(\n", - " path: Path,\n", - " skip: Union[str, List[str]]=None\n", - ") -> pd.DataFrame:\n", - " \"\"\"Get all file paths from a directory\"\"\"\n", - " path = Path(path).absolute()\n", - " files = []\n", - " if type(skip) == str:\n", - " def not_skipping(phrase)->bool:\n", - " return skip in phrase\n", - " elif type(skip) == list:\n", - " def not_skipping(phrase)->bool:\n", - " for sk in skip:\n", - " if sk in phrase:\n", - " return True\n", - " return False\n", - " elif skip is None:\n", - " not_skipping = lambda x:True\n", - " else:\n", - " raise TypeError(\n", - " f\"type of skip not correct: {type(skip)}\")\n", - "\n", - " def get_files_(p: Path) -> None:\n", - " if p.is_dir():\n", - " for i in p.ls():\n", - " if not_skipping(i):\n", - " get_files_(p/i)\n", - " else:\n", - " files.append(str(p))\n", - " get_files_(path)\n", - "\n", - " return pd.DataFrame(dict(path=files))\n", - "\n", - "def file_detail(\n", - " path: Path,\n", - " skip: Union[str, List[str]]=None\n", - ") -> pd.DataFrame:\n", - " \"\"\"\n", - " Get detailed file information from a directory\n", - " \"\"\"\n", - " file_df = get_files(path, skip=skip)\n", - " def find_type(x):\n", - " try:\n", - " x = Path(x)\n", - " if x.is_dir():\n", - " return \"directory\"\n", - " file_type = x.name.split(\".\")[-1]\n", - " if file_type == x.name:\n", - " return None\n", - " else:\n", - " return file_type\n", - " except Exception as e:\n", - " return None\n", - "\n", - " def find_parent(x):\n", - " try:\n", - " x = Path(x)\n", - " return x.parent.name\n", - " except Exception as e:\n", - " return None\n", - "\n", - " def find_depth(x):\n", - " try:\n", - " x = Path(x)\n", - " return len(x.parts)\n", - " except Exception as e:\n", - " return None\n", - " file_df[\"file_type\"] = file_df.path.apply(find_type)\n", - " file_df[\"parent\"] = file_df.path.apply(find_parent)\n", - " file_df[\"depth\"] = file_df.path.apply(find_depth)\n", - "\n", - " return file_df" - ] - }, - { - "cell_type": "code", - "execution_count": 35, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -135,152 +51,68 @@ " \n", " \n", " \n", - " 0\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " DS_Store\n", - " apiGateway-js-sdk\n", - " 12\n", - " \n", - " \n", - " 1\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " md\n", - " apiGateway-js-sdk\n", - " 12\n", - " \n", - " \n", - " 2\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " apiGateway-js-sdk\n", - " 12\n", - " \n", - " \n", - " 3\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " components\n", - " 15\n", - " \n", - " \n", - " 4\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " components\n", - " 15\n", - " \n", - " \n", - " 5\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " rollups\n", - " 15\n", - " \n", - " \n", - " 6\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " rollups\n", - " 15\n", - " \n", - " \n", - " 7\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " url-template\n", - " 14\n", - " \n", - " \n", - " 8\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " apiGatewayCore\n", - " 14\n", + " 0\n", + " /Users/salvor/github/forgebox/nbs/../../../Doc...\n", + " docx\n", + " Documents\n", + " 11\n", " \n", " \n", - " 9\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " apiGatewayCore\n", - " 14\n", + " 1\n", + " /Users/salvor/github/forgebox/nbs/../../../Doc...\n", + " DS_Store\n", + " Documents\n", + " 11\n", " \n", " \n", - " 10\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " apiGatewayCore\n", - " 14\n", + " 2\n", + " /Users/salvor/github/forgebox/nbs/../../../Doc...\n", + " localized\n", + " Documents\n", + " 11\n", " \n", " \n", + " 3\n", + " /Users/salvor/github/forgebox/nbs/../../../Doc...\n", + " xlsx\n", + " Documents\n", " 11\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " apiGatewayCore\n", - " 14\n", " \n", " \n", - " 12\n", - " /Users/xiaochen.zhang/github/forgebox/nbs/../....\n", - " js\n", - " dist\n", - " 15\n", + " 4\n", + " /Users/salvor/github/forgebox/nbs/../../../Doc...\n", + " xlsx\n", + " Documents\n", + " 11\n", " \n", " \n", "\n", "" ], "text/plain": [ - " path file_type \\\n", - "0 /Users/xiaochen.zhang/github/forgebox/nbs/../.... DS_Store \n", - "1 /Users/xiaochen.zhang/github/forgebox/nbs/../.... md \n", - "2 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", - "3 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", - "4 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", - "5 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", - "6 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", - "7 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", - "8 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", - "9 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", - "10 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", - "11 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", - "12 /Users/xiaochen.zhang/github/forgebox/nbs/../.... js \n", + " path ... depth\n", + "0 /Users/salvor/github/forgebox/nbs/../../../Doc... ... 11\n", + "1 /Users/salvor/github/forgebox/nbs/../../../Doc... ... 11\n", + "2 /Users/salvor/github/forgebox/nbs/../../../Doc... ... 11\n", + "3 /Users/salvor/github/forgebox/nbs/../../../Doc... ... 11\n", + "4 /Users/salvor/github/forgebox/nbs/../../../Doc... ... 11\n", "\n", - " parent depth \n", - "0 apiGateway-js-sdk 12 \n", - "1 apiGateway-js-sdk 12 \n", - "2 apiGateway-js-sdk 12 \n", - "3 components 15 \n", - "4 components 15 \n", - "5 rollups 15 \n", - "6 rollups 15 \n", - "7 url-template 14 \n", - "8 apiGatewayCore 14 \n", - "9 apiGatewayCore 14 \n", - "10 apiGatewayCore 14 \n", - "11 apiGatewayCore 14 \n", - "12 dist 15 " + "[5 rows x 4 columns]" ] }, - "execution_count": 35, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "file_detail(\"../../../Downloads/apiGateway-js-sdk\")" + "file_detail(\"../../../Documents/\").head()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/nbs/51_image_widgets.ipynb b/nbs/51_image_widgets.ipynb index 7a61fbc..371cc58 100644 --- a/nbs/51_image_widgets.ipynb +++ b/nbs/51_image_widgets.ipynb @@ -8,37 +8,6 @@ "> Extra widgets for jupyter noteook" ] }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp images.widgets" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "import base64\n", - "from io import BytesIO\n", - "from PIL import Image\n", - "from typing import List, Callable\n", - "import math\n", - "from forgebox.html import DOM, image_to_base64, data_url\n" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -55,129 +24,12 @@ }, { "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "# export\n", - "def image_dom(\n", - " img: Image,\n", - " **kwargs\n", - "):\n", - " \"\"\"\n", - " Create tag with src='data:...'\n", - " with PIL.Image object\n", - " return forgebox.html.DOM\n", - " \"\"\"\n", - " from forgebox.html import DOM\n", - "\n", - " img_dom = DOM(\"\", \"img\", dict(\n", - " src=data_url(img),\n", - " ))\n", - " kwargs.update({\"style\": \"padding:3px\"})\n", - " div_dom = DOM(\"\", \"div\", kwargs)\n", - " div_dom.append(img_dom)\n", - " return div_dom\n", - "\n", - "\n", - "def with_title_dom(img: Image, title:str, col: int):\n", - " from forgebox.html import DOM\n", - " img_dom = image_dom(img)\n", - " out_dom = DOM(img_dom, \"div\", {\"class\":f\"col-sm-{col}\"})\n", - " out_dom.append(title)\n", - " return out_dom\n", - "\n", - "\n", - "def view_images(\n", - " *images,\n", - " num_per_row: int=4,\n", - " titles: List=None,\n", - "):\n", - " \"\"\"\n", - " Create
wraping up images\n", - " view_images(\n", - " img1, img2, img3,\n", - " img4, img5, img6,\n", - " img7)()\n", - " \"\"\"\n", - " from forgebox.html import DOM\n", - " frame = DOM(\"\", \"div\", {\"class\": \"row\"})\n", - " assert num_per_row in [1, 2, 3, 4, 6, 12],\\\n", - " \"num_per_row should be in [1, 2, 3, 4, 6, 12]\"\n", - " col = 12//num_per_row\n", - "\n", - " if titles is not None:\n", - " assert len(titles) == len(images),\\\n", - " f\"title length:{len(titles)}!=image length:({len(images)})\"\n", - " if titles is None:\n", - " for img in images:\n", - " frame.append(image_dom(img, **{\"class\": f\"col-sm-{col}\"}))\n", - " else:\n", - " for img, title in zip(images, titles):\n", - " frame.append(with_title_dom(img, title, col))\n", - " return frame" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Subplot" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "# export\n", - "class Subplots:\n", - " \"\"\"\n", - " Simplifying plt sublots\n", - " sub = Subplots(18)\n", - "\n", - " ```\n", - " @sub.single\n", - " def plotting(ax, data):\n", - " ax.plot(data)\n", - "\n", - " for data in data_list:\n", - " sub(data)\n", - " ```\n", - " \"\"\"\n", - "\n", - " def __init__(self, total: int, ncol: int = 3, figsize=None):\n", - " \"\"\"\n", - " total:int, total plot numbers\n", - " ncol:int, number of columns\n", - " figsize: Tuple, default (15, vertical_items*4)\n", - " \"\"\"\n", - " from matplotlib import pyplot as plt\n", - "\n", - " self.i = 0\n", - " self.size = list([math.ceil(float(total)/float(ncol)), ncol])\n", - " self.ncol = ncol\n", - " self.total = total\n", - " self.figsize = figsize if figsize is not None else (15, self.size[0]*4)\n", - " self.fig, self.ax = plt.subplots(*self.size, figsize=self.figsize)\n", - "\n", - " def __len__(self) -> int: return self.total\n", - "\n", - " def single(self, f: Callable) -> Callable:\n", - " \"\"\"\n", - " Decorating a plt subplot function\n", - " \"\"\"\n", - " self.single_func = f\n", - " return self.single_func\n", - "\n", - " def __call__(self, *args, **kwargs):\n", - " if self.i >= self.total:\n", - " raise StopIteration(f\"You said total would be {self.total}!\")\n", - " ax = self.ax[math.floor(\n", - " float(self.i)/float(self.ncol)), self.i % self.ncol]\n", - " self.single_func(ax, *args, **kwargs)\n", - " self.i += 1" + "from forgebox.images.widgets import image_dom, with_title_dom, view_images, Subplots\n", + "import os" ] }, { @@ -187,15 +39,6 @@ "## Test on image preview" ] }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -345,7 +188,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/nbs/CUDA_GPU_Management.ipynb b/nbs/CUDA_GPU_Management.ipynb index dddb904..6b15f59 100644 --- a/nbs/CUDA_GPU_Management.ipynb +++ b/nbs/CUDA_GPU_Management.ipynb @@ -15,15 +15,6 @@ "## Handler on a single CUDA device" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# default_exp ftorch.cuda" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -40,122 +31,8 @@ "metadata": {}, "outputs": [], "source": [ - "# export \n", "import torch\n", - "import subprocess\n", - "\n", - "class CudaDevice(object):\n", - " def __init__(self, idx):\n", - " \"\"\"\n", - " idx: int, cuda device id\n", - " d = CudaDevice(0)\n", - " print(d)\n", - " \"\"\"\n", - " self.idx = idx\n", - " self.device = torch.device(idx)\n", - " if hasattr(self, \"used\") == False:\n", - " self.refresh()\n", - "\n", - " def __repr__(self):\n", - " return f\"Device {self.idx}: \\n\\tname:{self.prop.name}\\n\\tused:{self.used}MB\\tfree:{self.free}MB\"\n", - "\n", - " @property\n", - " def prop(self):\n", - " \"\"\"\n", - " property on cuda device\n", - " \"\"\"\n", - " return torch.cuda.get_device_properties(self.idx)\n", - "\n", - " def __len__(self):\n", - " return int(self.prop.total_memory)\n", - "\n", - " @property\n", - " def mem(self):\n", - " return f\"{self.__len__() // 1024 // 1024} MB\"\n", - "\n", - " def __int__(self):\n", - " return int(self.idx)\n", - "\n", - " def refresh(self):\n", - " gpu_stats = subprocess.check_output([\"nvidia-smi\", \"--format=csv\", \"--query-gpu=memory.used,memory.free\"])\n", - " stat = list(\n", - " int(v) for v in str(gpu_stats).split(\"\\\\n\")[1 + self.idx].replace(\" MiB\", \"\").replace(\" \", \"\").split(\",\"))\n", - " setattr(self, \"used\", stat[0])\n", - " setattr(self, \"free\", stat[1])\n", - " return stat\n", - "\n", - " def __call__(self):\n", - " return self.device" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## A handler managing multiple devices" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# export \n", - "class CudaHandler(object):\n", - " def __init__(self):\n", - " \"\"\"\n", - " ch = CudaHandler()\n", - " dev = ch.idle()()\n", - " some_torch_tensor.to(dev)\n", - " \"\"\"\n", - " if torch.cuda.is_available() == False:\n", - " return None\n", - " self.counts = torch.cuda.device_count()\n", - " print(f\">>> {self.counts} cuda devices found >>>\")\n", - " self.devices = list()\n", - " for i in range(self.counts):\n", - " d = CudaDevice(i)\n", - " self.devices.append(d)\n", - " print(f\"{d}\")\n", - " setattr(self, f\"gpu{i}\", d)\n", - "\n", - " def __len__(self):\n", - " \"\"\"\n", - " counts of the cuda devices numbers\n", - " \"\"\"\n", - " return self.counts\n", - "\n", - " def __getitem__(self, idx):\n", - " return self.devices[idx]\n", - "\n", - " def __repr__(self):\n", - " self.refresh()\n", - " return f\"<{self.counts} cuda devices>\\n\" + \"\\n\".join(list(str(d) for d in (self.devices)))\n", - "\n", - " def refresh(self):\n", - " \"\"\"\n", - " refresh the cuda mem stats\n", - " \"\"\"\n", - " gpu_stats = subprocess.check_output([\"nvidia-smi\", \"--format=csv\", \"--query-gpu=memory.used,memory.free\"])\n", - " rows = list(list(int(v) for v in vs.replace(\" MiB\", \"\").replace(\" \", \"\").split(\",\")) for vs in\n", - " str(gpu_stats).split(\"\\\\n\")[1:-1])\n", - " for i in range(len(rows)):\n", - " setattr(self.devices[i], \"used\", rows[i][0])\n", - " setattr(self.devices[i], \"free\", rows[i][1])\n", - " print(\"cuda stats refreshed\")\n", - "\n", - " def idle(self):\n", - " \"\"\"\n", - " find the most idle cuda device\n", - " \"\"\"\n", - " self.refresh()\n", - " rt = self[0]\n", - " for d in self.devices:\n", - " if d.free > rt.free:\n", - " rt = d\n", - " print(f\"Found the most idle GPU: cuda:{rt.idx}, {rt.free} MB Mem remained\")\n", - " return rt" + "from forgebox.ftorch.cuda import CudaDevice, CudaHandler, Cudas" ] }, { @@ -169,22 +46,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "These handler is to simplify the multi-cuda situation, where you can allocate the most idle GPU" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from forgebox.ftorch.cuda import CudaHandler" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ + "These handler is to simplify the multi-cuda situation, where you can allocate the most idle GPU\n", + "\n", "Substantiate the handler" ] }, @@ -232,7 +95,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -247,6 +110,19 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false } }, "nbformat": 4,