diff --git a/Orchestrator/docs/NLRModels.md b/Orchestrator/docs/NLRModels.md index 560e391ac..57987d6ef 100644 --- a/Orchestrator/docs/NLRModels.md +++ b/Orchestrator/docs/NLRModels.md @@ -52,16 +52,6 @@ This is a yet another high quality EN-only base model for entity extraction. It is a 12-layer pretrained pretrained [Transformer][7] model optimized for conversation. Its architecture is pretrained for example-based use ([KNN][3]), thus it can be used out of box. -### pretrained.20210105.microsoft.dte.00.12.bert_example_ner_multilingual.onnx (experimental) -This is a high quality multilingual base model for entity extraction. -It is a 12-layer pretrained pretrained [Transformer][7] model optimized for conversation. -Its architecture is pretrained for example-based use ([KNN][3]), thus it can be used out of box. - -### pretrained.20210105.microsoft.dte.00.12.tulr_example_ner_multilingual.onnx (experimental) -This is a high quality multilingual base model for entity extraction. -It is a 12-layer pretrained pretrained [Transformer][7] model optimized for conversation. -Its architecture is pretrained for example-based use ([KNN][3]), thus it can be used out of box. - ### pretrained.20210205.microsoft.dte.00.06.bert_example_ner.en.onnx (experimental) This is a high quality EN-only base model for entity extraction. It's smaller and faster than its 12-layer alternative. It is a 6-layer pretrained pretrained [Transformer][7] model optimized for conversation. @@ -72,16 +62,6 @@ This is a high quality EN-only base model for entity extraction. It's smaller an It is a 6-layer pretrained pretrained [Transformer][7] model optimized for conversation. Its architecture is pretrained for example-based use ([KNN][3]), thus it can be used out of box. -### pretrained.20210205.microsoft.dte.00.06.bert_example_ner_multilingual.onnx (experimental) -This is a high quality multilingual base model for entity extraction. It's smaller and faster than its 12-layer alternative. -It is a 6-layer pretrained pretrained [Transformer][7] model optimized for conversation. -Its architecture is pretrained for example-based use ([KNN][3]), thus it can be used out of box. - -### pretrained.20210205.microsoft.dte.00.06.tulr_example_ner_multilingual.onnx (experimental) -This is a high quality multilingual base model for entity extraction. It's smaller and faster than its 12-layer alternative. -It is a 6-layer pretrained pretrained [Transformer][7] model optimized for conversation. -Its architecture is pretrained for example-based use ([KNN][3]), thus it can be used out of box. - ## Models Evaluation For a more quantitative comparison analysis of the different models see the following performance characteristics. @@ -136,13 +116,17 @@ For a more quantitative comparison analysis of the different models see the foll | ------------------------------------------------------------ | ---------- | ------ | ----------------------- | --------------- | | pretrained.20210205.microsoft.dte.00.06.bert_example_ner.en.onnx | BERT | 6 | ~ 23 ms | 259M | | pretrained.20210205.microsoft.dte.00.12.bert_example_ner.en.onnx | BERT | 12 | ~ 40 ms | 427M | +| pretrained.20210218.microsoft.dte.00.06.bert_example_ner.en.onnx | BERT | 6 | ~ 23 ms | 259M | +| pretrained.20210218.microsoft.dte.00.12.bert_example_ner.en.onnx | BERT | 12 | ~ 40 ms | 425M | - The following table shows how accurate is each model relative to provided training sample size using [Snips NLU][4] system, evaluated by **macro-average-F1**. | Training samples per entity type | 10 | 20 | 50 | 100 | 200 | | ------------------------------------------------------------ | ----- | ----- | ----- | ----- | ----- | -| pretrained.20210205.microsoft.dte.00.06.bert_example_ner.en.onnx | 0.662 | 0.678 | 0.680 | 0.684 | 0.674 | +| pretrained.20210205.microsoft.dte.00.06.bert_example_ner.en.onnx | 0.615 | 0.636 | 0.647 | 0.661 | 0.665 | | pretrained.20210205.microsoft.dte.00.12.bert_example_ner.en.onnx | 0.637 | 0.658 | 0.684 | 0.698 | 0.702 | +| pretrained.20210218.microsoft.dte.00.06.bert_example_ner.en.onnx | 0.637 | 0.658 | 0.673 | 0.686 | 0.684 | +| pretrained.20210218.microsoft.dte.00.12.bert_example_ner.en.onnx | 0.661 | 0.664 | 0.670 | 0.685 | 0.681 | diff --git a/Orchestrator/v0.2/nlr_versions.json b/Orchestrator/v0.2/nlr_versions.json index 7df25f7df..412b970e9 100644 --- a/Orchestrator/v0.2/nlr_versions.json +++ b/Orchestrator/v0.2/nlr_versions.json @@ -41,18 +41,6 @@ "description": "Bot Framework SDK release 4.10 - Multilingual ONNX V1.4 12-layer per-token intent base model", "minSDKVersion": "4.10.0" }, - "pretrained.20210105.microsoft.dte.00.12.bert_example_ner_multilingual.onnx": { - "releaseDate": "01/05/2021", - "modelUri": "https://models.botframework.com/models/dte/onnx/pretrained.20210105.microsoft.dte.00.12.bert_example_ner_multilingual.onnx.zip", - "description": "(experimental) Bot Framework SDK release 4.10 - Multilingual ONNX V1.4 12-layer per-token entity base model", - "minSDKVersion": "4.10.0" - }, - "pretrained.20210105.microsoft.dte.00.12.tulr_example_ner_multilingual.onnx": { - "releaseDate": "01/05/2021", - "modelUri": "https://models.botframework.com/models/dte/onnx/pretrained.20210105.microsoft.dte.00.12.tulr_example_ner_multilingual.onnx.zip", - "description": "(experimental) Bot Framework SDK release 4.10 - Multilingual ONNX V1.4 12-layer per-token entity base model", - "minSDKVersion": "4.10.0" - }, "pretrained.20210205.microsoft.dte.00.06.bert_example_ner.en.onnx": { "releaseDate": "02/05/2021", "modelUri": "https://models.botframework.com/models/dte/onnx/pretrained.20210205.microsoft.dte.00.06.bert_example_ner.en.onnx.zip", @@ -70,18 +58,6 @@ "modelUri": "https://models.botframework.com/models/dte/onnx/pretrained.20210205.microsoft.dte.00.06.unicoder_multilingual.onnx.zip", "description": "Bot Framework SDK release 4.10 - Multilingual ONNX V1.4 6-layer per-token intent base model", "minSDKVersion": "4.10.0" - }, - "pretrained.20210205.microsoft.dte.00.06.bert_example_ner_multilingual.onnx": { - "releaseDate": "02/05/2021", - "modelUri": "https://models.botframework.com/models/dte/onnx/pretrained.20210205.microsoft.dte.00.06.bert_example_ner_multilingual.onnx.zip", - "description": "(experimental) Bot Framework SDK release 4.10 - Multilingual ONNX V1.4 6-layer per-token entity base model", - "minSDKVersion": "4.10.0" - }, - "pretrained.20210205.microsoft.dte.00.06.tulr_example_ner_multilingual.onnx": { - "releaseDate": "02/05/2021", - "modelUri": "https://models.botframework.com/models/dte/onnx/pretrained.20210205.microsoft.dte.00.06.tulr_example_ner_multilingual.onnx.zip", - "description": "(experimental) Bot Framework SDK release 4.10 - Multilingual ONNX V1.4 6-layer per-token entity base model", - "minSDKVersion": "4.10.0" } } }