WIP: SB Investigation

.gitignore

@@ -124,6 +124,7 @@ celerybeat.pid
 .venv
 env/
 venv/
+venv3.8/
 ENV/
 env.bak/
 venv.bak/
@@ -157,4 +158,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+#.idea/
+
+# make sure no test data accidentially included
+data/*

clu_export.py

@@ -0,0 +1,44 @@
+"""
+python clu_export.py
+
+A way of testing locally conversion of a passed JSON into CLU format
+
+"""
+# ******************************************************************************************************************120
+
+# standard imports
+import json
+
+# 3rd party imports
+import click
+
+# custom imports
+import hf_integration.clu_converters
+
+@click.command()
+@click.option('-f', '--filename', type=str, required=True, help='Input HF JSON')
+@click.option('-m', '--merge_filename', type=str, required=True, help='Input CLU to merge with')
+def main(filename: str, merge_filename:str) -> None: # pylint: disable=unused-argument
+    """Main Function"""
+
+    # read the json
+    with open(filename,mode="r",encoding="utf8") as file_in:
+        hf_dict = json.load(file_in)
+
+    with open(merge_filename,mode="r",encoding="utf8") as file_in_merge:
+        clu_dict = json.load(file_in_merge)
+
+    converter = hf_integration.clu_converters.hf_to_clu_converter()
+    hf_dict = converter.hf_to_clu_process(hf_json=hf_dict,clu_json=clu_dict,delimiter="-",language="nl")
+
+    # write the file
+    output_filename = filename.replace(".json","_clu_output.json")
+    assert output_filename != filename
+    with open(output_filename,mode="w",encoding="utf8") as file_out:
+        json.dump(hf_dict,file_out,indent=4)
+        print(f'Wrote to: {output_filename}')
+
+
+
+if __name__ == '__main__':
+    main()  # pylint: disable=no-value-for-parameter

clu_import.py

@@ -0,0 +1,40 @@
+"""
+python clu_import.py
+
+A way of testing locally conversion of a passed JSON into HF format
+
+"""
+# ******************************************************************************************************************120
+
+# standard imports
+import json
+
+# 3rd party imports
+import click
+
+# custom imports
+import hf_integration.clu_converters
+
+@click.command()
+@click.option('-f', '--filename', type=str, required=True, help='Input CLU JSON')
+def main(filename: str) -> None: # pylint: disable=unused-argument
+    """Main Function"""
+
+    # read the json
+    with open(filename,mode="r",encoding="utf8") as file_in:
+        clu_dict = json.load(file_in)
+
+    converter = hf_integration.clu_converters.clu_to_hf_converter()
+    hf_dict = converter.clu_to_hf_process(clu_json=clu_dict,delimiter="-",language="nl")
+
+    # write the file
+    output_filename = filename.replace(".json","_hf_output.json")
+    assert output_filename != filename
+    with open(output_filename,mode="w",encoding="utf8") as file_out:
+        json.dump(hf_dict,file_out,indent=4)
+        print(f'Wrote to: {output_filename}')
+
+
+
+if __name__ == '__main__':
+    main()  # pylint: disable=no-value-for-parameter