Move scripts inside package for easier access

Also updates the installer to make resources and console
execution easier

scripts/finetune.py → alexa_teacher_models/scripts/finetune.py

@@ -20,7 +20,7 @@
 from dataclasses import dataclass, field
 from typing import Optional
 import numpy as np
-from train_utils import load_data, create_tokenizer, Preprocessor, setup_logging
+from alexa_teacher_models.scripts.train_utils import load_data, create_tokenizer, Preprocessor, setup_logging
 from metrics import get_metric
 
 # This allows us to us Auto* from HuggingFace to get to the model

scripts/preproc.py → alexa_teacher_models/scripts/preproc.py

@@ -16,7 +16,7 @@
 from dataclasses import dataclass, field
 from typing import Optional
 from datasets import DatasetDict
-from train_utils import load_data, create_tokenizer, Preprocessor
+from alexa_teacher_models.scripts.train_utils import load_data, create_tokenizer, Preprocessor
 
 # This allows us to us Auto* from HuggingFace to get to the model
 import alexa_teacher_models

scripts/repl.py → alexa_teacher_models/scripts/repl.py

@@ -21,7 +21,7 @@
 import logging
 from pprint import pprint
 from prompt_toolkit import PromptSession
-from train_utils import create_tokenizer
+from alexa_teacher_models.scripts.train_utils import create_tokenizer
 
 DEFAULT_PRE = '[CLM] '
 # TODO: make configurable level

setup.cfg

@@ -4,7 +4,7 @@ version = 1.0.0
 description = Alexa Teacher Models
 author = amazon-alexa 
 author_email = [email protected]
-url = https://www.github.com/https://github.com/alexa/alexa-teacher-models
+url = https://github.com/alexa/alexa-teacher-models
 license = Apache 2.0
 keywords=
     deep-learning
@@ -34,14 +34,17 @@ classifiers =
 
 [options]
 packages = find:
-
+include_package_data = True
 install_requires =
     numpy
     transformers
     sentencepiece>=0.1.95
-    torch>=1.7
+    torch>=1.7,<2.0
     protobuf<3.21.0
 
+[options.package_data]
+alexa_teacher_models = scripts/deepspeed/*.json
+
 [options.extras_require]
 repl:
     prompt_toolkit