Merge pull request #156 from KennethEnevoldsen/fix_instruct_tuned_embed

fix: Pass the task for encode_queries, and encode_corpus

src/seb/cache/intfloat__multilingual-e5-large-instruct/Angry_Tweets.json

@@ -1 +1 @@
-{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1","time_of_run":"2024-02-19T09:59:32.444928","scores":{"da":{"accuracy":0.6385864374403056,"f1":0.6346397415495485,"accuracy_stderr":0.008693073736758542,"f1_stderr":0.008949873953394303,"main_score":0.6385864374403056}},"main_score":"accuracy"}
+{"task_name":"Angry Tweets","task_description":"A sentiment dataset with 3 classes (positiv, negativ, neutral) for Danish tweets","task_version":"1.1.1","time_of_run":"2024-02-19T16:34:45.535603","scores":{"da":{"accuracy":0.6457497612225407,"f1":0.6418225953633901,"accuracy_stderr":0.011740083314233166,"f1_stderr":0.010452977466255427,"main_score":0.6457497612225407}},"main_score":"accuracy"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/Bornholm_Parallel.json

@@ -1 +1 @@
-{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1","time_of_run":"2024-02-19T10:00:36.624049","scores":{"da":{"precision":0.5127666666666667,"recall":0.61,"f1":0.5404380952380953,"accuracy":0.61,"main_score":0.5404380952380953},"da-bornholm":{"precision":0.5127666666666667,"recall":0.61,"f1":0.5404380952380953,"accuracy":0.61,"main_score":0.5404380952380953}},"main_score":"f1"}
+{"task_name":"Bornholm Parallel","task_description":"Danish Bornholmsk Parallel Corpus. Bornholmsk is a Danish dialect spoken on the island of Bornholm, Denmark. Historically it is a part of east Danish which was also spoken in Scania and Halland, Sweden.","task_version":"1.1.1","time_of_run":"2024-02-19T16:36:39.75597","scores":{"da":{"precision":0.5210857142857143,"recall":0.62,"f1":0.5502333333333334,"accuracy":0.62,"main_score":0.5502333333333334},"da-bornholm":{"precision":0.5210857142857143,"recall":0.62,"f1":0.5502333333333334,"accuracy":0.62,"main_score":0.5502333333333334}},"main_score":"f1"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/DKHate.json

@@ -1 +1 @@
-{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1","time_of_run":"2024-02-19T10:18:13.73009","scores":{"da":{"accuracy":0.6714285714285715,"f1":0.5528429107551277,"ap":0.18912292120124466,"accuracy_stderr":0.07407251803066195,"f1_stderr":0.062100582121016365,"ap_stderr":0.03956920812193161,"main_score":0.6714285714285715}},"main_score":"accuracy"}
+{"task_name":"DKHate","task_description":"Danish Tweets annotated for Hate Speech either being Offensive or not","task_version":"1.1.1","time_of_run":"2024-02-19T16:38:58.791777","scores":{"da":{"accuracy":0.6714285714285715,"f1":0.5528429107551277,"ap":0.18912292120124466,"accuracy_stderr":0.07407251803066195,"f1_stderr":0.062100582121016365,"ap_stderr":0.03956920812193161,"main_score":0.6714285714285715}},"main_score":"accuracy"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/Da_Political_Comments.json

@@ -1 +1 @@
-{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1","time_of_run":"2024-02-19T10:21:03.628142","scores":{"da":{"accuracy":0.4533296337402885,"f1":0.42926373854308253,"accuracy_stderr":0.026201641326856345,"f1_stderr":0.022752012723108207,"main_score":0.4533296337402885}},"main_score":"accuracy"}
+{"task_name":"Da Political Comments","task_description":"A dataset of Danish political comments rated for sentiment","task_version":"1.1.1","time_of_run":"2024-02-19T16:41:49.159723","scores":{"da":{"accuracy":0.4533296337402885,"f1":0.42926373854308253,"accuracy_stderr":0.026201641326856345,"f1_stderr":0.022752012723108207,"main_score":0.4533296337402885}},"main_score":"accuracy"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/DanFEVER.json

@@ -1 +1 @@
-{"task_name":"DanFEVER","task_description":"A Danish dataset intended for misinformation research. It follows the same format as the English FEVER dataset.","task_version":"1.1.1","time_of_run":"2024-02-19T10:29:38.450736","scores":{"da":{"ndcg_at_1":0.26393,"ndcg_at_3":0.3431,"ndcg_at_5":0.35529,"ndcg_at_10":0.36361,"ndcg_at_100":0.37006,"ndcg_at_1000":0.37046,"map_at_1":0.26385,"map_at_3":0.32447,"map_at_5":0.33126,"map_at_10":0.33473,"map_at_100":0.33621,"map_at_1000":0.33623,"recall_at_1":0.26385,"recall_at_3":0.39667,"recall_at_5":0.42609,"recall_at_10":0.45167,"recall_at_100":0.48031,"recall_at_1000":0.48329,"precision_at_1":0.26393,"precision_at_3":0.13228,"precision_at_5":0.08527,"precision_at_10":0.04519,"precision_at_100":0.00481,"precision_at_1000":0.00048,"mrr_at_1":0.26393,"mrr_at_3":0.32447,"mrr_at_5":0.33128,"mrr_at_10":0.33475,"mrr_at_100":0.33622,"mrr_at_1000":0.33623}},"main_score":"ndcg_at_10"}
+{"task_name":"DanFEVER","task_description":"A Danish dataset intended for misinformation research. It follows the same format as the English FEVER dataset.","task_version":"1.1.1","time_of_run":"2024-02-19T17:40:45.928545","scores":{"da":{"ndcg_at_1":0.29562,"ndcg_at_3":0.38425,"ndcg_at_5":0.39156,"ndcg_at_10":0.39523,"ndcg_at_100":0.39805,"ndcg_at_1000":0.3981,"map_at_1":0.29554,"map_at_3":0.36397,"map_at_5":0.36807,"map_at_10":0.36965,"map_at_100":0.37033,"map_at_1000":0.37033,"recall_at_1":0.29554,"recall_at_3":0.44226,"recall_at_5":0.45975,"recall_at_10":0.47081,"recall_at_100":0.48298,"recall_at_1000":0.48329,"precision_at_1":0.29562,"precision_at_3":0.1475,"precision_at_5":0.09201,"precision_at_10":0.04712,"precision_at_100":0.00483,"precision_at_1000":0.00048,"mrr_at_1":0.29562,"mrr_at_3":0.36404,"mrr_at_5":0.36815,"mrr_at_10":0.36971,"mrr_at_100":0.37039,"mrr_at_1000":0.37039}},"main_score":"ndcg_at_10"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/LCC.json

@@ -1 +1 @@
-{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1","time_of_run":"2024-02-19T09:54:40.009147","scores":{"da":{"accuracy":0.704,"f1":0.701848194200893,"accuracy_stderr":0.03143953632680427,"f1_stderr":0.02902016662197141,"main_score":0.704}},"main_score":"accuracy"}
+{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1","time_of_run":"2024-02-19T16:35:32.435049","scores":{"da":{"accuracy":0.7060000000000001,"f1":0.7044016863909317,"accuracy_stderr":0.03405224743765903,"f1_stderr":0.02958320828055799,"main_score":0.7060000000000001}},"main_score":"accuracy"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/NorQuad.json

@@ -1 +1 @@
-{"task_name":"NorQuad","task_description":"Human-created question for Norwegian wikipedia passages.","task_version":"0.0.1","time_of_run":"2024-02-19T13:15:30.065064","scores":{"nb":{"ndcg_at_1":0.24805,"ndcg_at_3":0.20309,"ndcg_at_5":0.2192,"ndcg_at_10":0.2405,"ndcg_at_100":0.28552,"ndcg_at_1000":0.32656,"map_at_1":0.12402,"map_at_3":0.1604,"map_at_5":0.16926,"map_at_10":0.17741,"map_at_100":0.18514,"map_at_1000":0.18653,"recall_at_1":0.12402,"recall_at_3":0.19531,"recall_at_5":0.22705,"recall_at_10":0.28076,"recall_at_100":0.46387,"recall_at_1000":0.74902,"precision_at_1":0.24805,"precision_at_3":0.13021,"precision_at_5":0.09082,"precision_at_10":0.05615,"precision_at_100":0.00928,"precision_at_1000":0.0015,"mrr_at_1":0.24805,"mrr_at_3":0.30062,"mrr_at_5":0.31219,"mrr_at_10":0.32457,"mrr_at_100":0.33511,"mrr_at_1000":0.33593}},"main_score":"ndcg_at_10"}
+{"task_name":"NorQuad","task_description":"Human-created question for Norwegian wikipedia passages.","task_version":"0.0.1","time_of_run":"2024-02-19T17:22:59.634924","scores":{"nb":{"ndcg_at_1":0.73047,"ndcg_at_3":0.5393,"ndcg_at_5":0.55794,"ndcg_at_10":0.57974,"ndcg_at_100":0.6218,"ndcg_at_1000":0.65057,"map_at_1":0.36523,"map_at_3":0.45199,"map_at_5":0.46468,"map_at_10":0.47643,"map_at_100":0.48799,"map_at_1000":0.48943,"recall_at_1":0.36523,"recall_at_3":0.4917,"recall_at_5":0.52832,"recall_at_10":0.5835,"recall_at_100":0.75146,"recall_at_1000":0.94385,"precision_at_1":0.73047,"precision_at_3":0.3278,"precision_at_5":0.21133,"precision_at_10":0.1167,"precision_at_100":0.01503,"precision_at_1000":0.00189,"mrr_at_1":0.73047,"mrr_at_3":0.7832,"mrr_at_5":0.7918,"mrr_at_10":0.79751,"mrr_at_100":0.80017,"mrr_at_1000":0.80025}},"main_score":"ndcg_at_10"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/SNL_Retrieval.json

@@ -1 +1 @@
-{"task_name":"SNL Retrieval","task_description":"Webscrabed articles and ingresses from the Norwegian lexicon 'Det Store Norske Leksikon'.","task_version":"0.0.1","time_of_run":"2024-02-19T13:13:48.683022","scores":{"nb":{"ndcg_at_1":0.84923,"ndcg_at_3":0.89391,"ndcg_at_5":0.90152,"ndcg_at_10":0.90796,"ndcg_at_100":0.9139,"ndcg_at_1000":0.91491,"map_at_1":0.84923,"map_at_3":0.88372,"map_at_5":0.88795,"map_at_10":0.89059,"map_at_100":0.89163,"map_at_1000":0.89167,"recall_at_1":0.84923,"recall_at_3":0.92308,"recall_at_5":0.94154,"recall_at_10":0.96154,"recall_at_100":0.99154,"recall_at_1000":0.99923,"precision_at_1":0.84923,"precision_at_3":0.30769,"precision_at_5":0.18831,"precision_at_10":0.09615,"precision_at_100":0.00992,"precision_at_1000":0.001,"mrr_at_1":0.84923,"mrr_at_3":0.88372,"mrr_at_5":0.88795,"mrr_at_10":0.89059,"mrr_at_100":0.89163,"mrr_at_1000":0.89167}},"main_score":"ndcg_at_10"}
+{"task_name":"SNL Retrieval","task_description":"Webscrabed articles and ingresses from the Norwegian lexicon 'Det Store Norske Leksikon'.","task_version":"0.0.1","time_of_run":"2024-02-19T17:31:30.962736","scores":{"nb":{"ndcg_at_1":0.93077,"ndcg_at_3":0.95189,"ndcg_at_5":0.95599,"ndcg_at_10":0.95753,"ndcg_at_100":0.96005,"ndcg_at_1000":0.96086,"map_at_1":0.93077,"map_at_3":0.94692,"map_at_5":0.94919,"map_at_10":0.94986,"map_at_100":0.95027,"map_at_1000":0.9503,"recall_at_1":0.93077,"recall_at_3":0.96615,"recall_at_5":0.97615,"recall_at_10":0.98077,"recall_at_100":0.99385,"recall_at_1000":1.0,"precision_at_1":0.93077,"precision_at_3":0.32205,"precision_at_5":0.19523,"precision_at_10":0.09808,"precision_at_100":0.00994,"precision_at_1000":0.001,"mrr_at_1":0.93077,"mrr_at_3":0.94692,"mrr_at_5":0.94919,"mrr_at_10":0.94986,"mrr_at_100":0.95027,"mrr_at_1000":0.9503}},"main_score":"ndcg_at_10"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/SweFAQ.json

@@ -1 +1 @@
-{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2024-02-19T13:29:06.137273","scores":{"sv":{"ndcg_at_1":0.54386,"ndcg_at_3":0.68514,"ndcg_at_5":0.71685,"ndcg_at_10":0.74194,"ndcg_at_100":0.75604,"ndcg_at_1000":0.75604,"map_at_1":0.54386,"map_at_3":0.65042,"map_at_5":0.66826,"map_at_10":0.67891,"map_at_100":0.68219,"map_at_1000":0.68219,"recall_at_1":0.54386,"recall_at_3":0.78558,"recall_at_5":0.8616,"recall_at_10":0.93762,"recall_at_100":1.0,"recall_at_1000":1.0,"precision_at_1":0.54386,"precision_at_3":0.26186,"precision_at_5":0.17232,"precision_at_10":0.09376,"precision_at_100":0.01,"precision_at_1000":0.001,"mrr_at_1":0.54386,"mrr_at_3":0.65042,"mrr_at_5":0.66826,"mrr_at_10":0.67891,"mrr_at_100":0.68219,"mrr_at_1000":0.68219}},"main_score":"ndcg_at_10"}
+{"task_name":"SweFAQ","task_description":"A Swedish QA dataset derived from FAQ","task_version":"0.0.1","time_of_run":"2024-02-19T17:56:10.929757","scores":{"sv":{"ndcg_at_1":0.60234,"ndcg_at_3":0.70512,"ndcg_at_5":0.73163,"ndcg_at_10":0.75417,"ndcg_at_100":0.7702,"ndcg_at_1000":0.77204,"map_at_1":0.60234,"map_at_3":0.67999,"map_at_5":0.6947,"map_at_10":0.70392,"map_at_100":0.70729,"map_at_1000":0.70737,"recall_at_1":0.60234,"recall_at_3":0.77778,"recall_at_5":0.84211,"recall_at_10":0.91228,"recall_at_100":0.98635,"recall_at_1000":1.0,"precision_at_1":0.60234,"precision_at_3":0.25926,"precision_at_5":0.16842,"precision_at_10":0.09123,"precision_at_100":0.00986,"precision_at_1000":0.001,"mrr_at_1":0.60234,"mrr_at_3":0.67999,"mrr_at_5":0.6947,"mrr_at_10":0.70392,"mrr_at_100":0.70729,"mrr_at_1000":0.70737}},"main_score":"ndcg_at_10"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/SwednRetrieval.json

@@ -1 +1 @@
-{"task_name":"SwednRetrieval","task_description":"News Article Summary Semantic Similarity Estimation.","task_version":"0.0.1","time_of_run":"2024-02-19T13:40:35.557813","scores":{"sv":{"ndcg_at_1":0.7334,"ndcg_at_3":0.69618,"ndcg_at_5":0.73429,"ndcg_at_10":0.75895,"ndcg_at_100":0.78753,"ndcg_at_1000":0.79532,"map_at_1":0.3667,"map_at_3":0.65316,"map_at_5":0.68377,"map_at_10":0.69827,"map_at_100":0.70676,"map_at_1000":0.70714,"recall_at_1":0.3667,"recall_at_3":0.69775,"recall_at_5":0.77246,"recall_at_10":0.83447,"recall_at_100":0.94238,"recall_at_1000":0.99316,"precision_at_1":0.7334,"precision_at_3":0.46517,"precision_at_5":0.30898,"precision_at_10":0.16689,"precision_at_100":0.01885,"precision_at_1000":0.00199,"mrr_at_1":0.7334,"mrr_at_3":0.78255,"mrr_at_5":0.79105,"mrr_at_10":0.79535,"mrr_at_100":0.79799,"mrr_at_1000":0.79813}},"main_score":"ndcg_at_10"}
+{"task_name":"SwednRetrieval","task_description":"News Article Summary Semantic Similarity Estimation.","task_version":"0.0.1","time_of_run":"2024-02-19T17:53:58.275729","scores":{"sv":{"ndcg_at_1":0.75195,"ndcg_at_3":0.70083,"ndcg_at_5":0.73671,"ndcg_at_10":0.7587,"ndcg_at_100":0.78775,"ndcg_at_1000":0.79562,"map_at_1":0.37598,"map_at_3":0.65251,"map_at_5":0.68212,"map_at_10":0.69546,"map_at_100":0.70379,"map_at_1000":0.70419,"recall_at_1":0.37598,"recall_at_3":0.69824,"recall_at_5":0.76855,"recall_at_10":0.82373,"recall_at_100":0.93555,"recall_at_1000":0.98682,"precision_at_1":0.75195,"precision_at_3":0.46549,"precision_at_5":0.30742,"precision_at_10":0.16475,"precision_at_100":0.01871,"precision_at_1000":0.00197,"mrr_at_1":0.75195,"mrr_at_3":0.80371,"mrr_at_5":0.81011,"mrr_at_10":0.81323,"mrr_at_100":0.81562,"mrr_at_1000":0.81572}},"main_score":"ndcg_at_10"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/TV2Nord_Retrieval.json

@@ -1 +1 @@
-{"task_name":"TV2Nord Retrieval","task_description":"News Article and corresponding summaries extracted from the Danish newspaper TV2 Nord.","task_version":"1.1.1","time_of_run":"2024-02-19T10:46:04.834117","scores":{"da":{"ndcg_at_1":0.89551,"ndcg_at_3":0.92811,"ndcg_at_5":0.93472,"ndcg_at_10":0.93807,"ndcg_at_100":0.94265,"ndcg_at_1000":0.9431,"map_at_1":0.89551,"map_at_3":0.92057,"map_at_5":0.92424,"map_at_10":0.92563,"map_at_100":0.92666,"map_at_1000":0.92668,"recall_at_1":0.89551,"recall_at_3":0.94971,"recall_at_5":0.96582,"recall_at_10":0.97607,"recall_at_100":0.99658,"recall_at_1000":1.0,"precision_at_1":0.89551,"precision_at_3":0.31657,"precision_at_5":0.19316,"precision_at_10":0.09761,"precision_at_100":0.00997,"precision_at_1000":0.001,"mrr_at_1":0.89551,"mrr_at_3":0.92057,"mrr_at_5":0.92424,"mrr_at_10":0.92563,"mrr_at_100":0.92666,"mrr_at_1000":0.92668}},"main_score":"ndcg_at_10"}
+{"task_name":"TV2Nord Retrieval","task_description":"News Article and corresponding summaries extracted from the Danish newspaper TV2 Nord.","task_version":"1.1.1","time_of_run":"2024-02-19T18:12:02.966688","scores":{"da":{"ndcg_at_1":0.89258,"ndcg_at_3":0.92787,"ndcg_at_5":0.93189,"ndcg_at_10":0.9369,"ndcg_at_100":0.94071,"ndcg_at_1000":0.9413,"map_at_1":0.89258,"map_at_3":0.9196,"map_at_5":0.92182,"map_at_10":0.92387,"map_at_100":0.92464,"map_at_1000":0.92466,"recall_at_1":0.89258,"recall_at_3":0.95166,"recall_at_5":0.96143,"recall_at_10":0.97705,"recall_at_100":0.99512,"recall_at_1000":0.99951,"precision_at_1":0.89258,"precision_at_3":0.31722,"precision_at_5":0.19229,"precision_at_10":0.09771,"precision_at_100":0.00995,"precision_at_1000":0.001,"mrr_at_1":0.89258,"mrr_at_3":0.9196,"mrr_at_5":0.92182,"mrr_at_10":0.92387,"mrr_at_100":0.92464,"mrr_at_1000":0.92466}},"main_score":"ndcg_at_10"}

src/seb/cache/intfloat__multilingual-e5-large-instruct/Twitterhjerne.json

@@ -1 +1 @@
-{"task_name":"Twitterhjerne","task_description":"Danish question asked on Twitter with the Hashtag #Twitterhjerne ('Twitter brain') and their corresponding answer.","task_version":"1.1.1","time_of_run":"2024-02-19T10:46:41.412894","scores":{"da":{"ndcg_at_1":0.78205,"ndcg_at_3":0.70396,"ndcg_at_5":0.70598,"ndcg_at_10":0.75463,"ndcg_at_100":0.809,"ndcg_at_1000":0.8113,"map_at_1":0.23782,"map_at_3":0.52628,"map_at_5":0.61611,"map_at_10":0.66177,"map_at_100":0.68691,"map_at_1000":0.68722,"recall_at_1":0.23782,"recall_at_3":0.56581,"recall_at_5":0.69017,"recall_at_10":0.8015,"recall_at_100":0.97607,"recall_at_1000":0.98718,"precision_at_1":0.78205,"precision_at_3":0.63248,"precision_at_5":0.47692,"precision_at_10":0.27564,"precision_at_100":0.03321,"precision_at_1000":0.00336,"mrr_at_1":0.78205,"mrr_at_3":0.84829,"mrr_at_5":0.84829,"mrr_at_10":0.85172,"mrr_at_100":0.85316,"mrr_at_1000":0.85316}},"main_score":"ndcg_at_10"}
+{"task_name":"Twitterhjerne","task_description":"Danish question asked on Twitter with the Hashtag #Twitterhjerne ('Twitter brain') and their corresponding answer.","task_version":"1.1.1","time_of_run":"2024-02-19T17:41:22.70945","scores":{"da":{"ndcg_at_1":0.84615,"ndcg_at_3":0.7423,"ndcg_at_5":0.74078,"ndcg_at_10":0.77233,"ndcg_at_100":0.82097,"ndcg_at_1000":0.8286,"map_at_1":0.25962,"map_at_3":0.56859,"map_at_5":0.66357,"map_at_10":0.69704,"map_at_100":0.71758,"map_at_1000":0.71833,"recall_at_1":0.25962,"recall_at_3":0.5938,"recall_at_5":0.71816,"recall_at_10":0.79188,"recall_at_100":0.94466,"recall_at_1000":0.98718,"precision_at_1":0.84615,"precision_at_3":0.65812,"precision_at_5":0.49487,"precision_at_10":0.27436,"precision_at_100":0.03256,"precision_at_1000":0.00336,"mrr_at_1":0.84615,"mrr_at_3":0.86538,"mrr_at_5":0.87436,"mrr_at_10":0.87619,"mrr_at_100":0.87834,"mrr_at_1000":0.87844}},"main_score":"ndcg_at_10"}

src/seb/interfaces/mteb_task.py

@@ -106,6 +106,17 @@ def evaluate(self, model: Encoder) -> TaskResult:
         split = self.mteb_task.description["eval_splits"][0]
         # Infusing task into encode()
         original_encode = model.encode
+
+        has_encode_queries = hasattr(model, "encode_queries")
+        has_encode_corpus = hasattr(model, "encode_corpus")
+
+        if has_encode_queries:
+            original_encode_queries = model.encode_queries  # type: ignore
+            model.encode_queries = partial(model.encode_queries, task=self)  # type: ignore
+        if has_encode_corpus:
+            original_encode_corpus = model.encode_corpus  # type: ignore
+            model.encode_corpus = partial(model.encode_corpus, task=self)  # type: ignore
+
         try:
             model.encode = partial(model.encode, task=self)
             scores = self.mteb_task.evaluate(model, split=split)
@@ -114,6 +125,11 @@ def evaluate(self, model: Encoder) -> TaskResult:
         finally:
             # Resetting encode to original
             model.encode = original_encode
+            if has_encode_queries:
+                model.encode_queries = original_encode_queries  # type: ignore
+            if has_encode_corpus:
+                model.encode_corpus = original_encode_corpus  # type: ignore
+
         if scores is None:
             raise ValueError("MTEBTask evaluation failed.")
 

src/seb/registered_models/e5_instruct_models.py

@@ -79,10 +79,10 @@ def task_to_instruction(task: Task) -> str:
     if task.task_type in ["Retrieval"]:
         task_name_to_instruct: dict[str, str] = {
             "Twitterhjerne": "Retrieve answers to questions asked in Danish tweets",
-            "SwednRetrieval": "Retrieve summaries of Swedish news articles",
-            "TV2Nord Retrieval": "Retrieve summaries of Danish news articles",
-            "DanFEVER": "Given a claim in Danish, retrieve documents that support or refute the claim",
-            "SNL Retrieval": "Given a lexicon article in Norwegian, retrieve its headline",
+            "SwednRetrieval": "Given a Swedish news headline retrieve summaries or news articles",
+            "TV2Nord Retrieval": "Given a summary of a Danish news article retrieve the corresponding news article",
+            "DanFEVER": "Given a claim in Danish, retrieve documents that support the claim",
+            "SNL Retrieval": "Given a lexicon headline in Norwegian, retrieve its article",
             "NorQuad": "Given a question in Norwegian, retrieve the answer from Wikipedia articles",
             "SweFAQ": "Retrieve answers given questions in Swedish",
             "ArguAna": "Given a claim, find documents that refute the claim",