From f6a49aa81b532383f6aa1044c7cd6265954b53c1 Mon Sep 17 00:00:00 2001
From: Brian Hie <blhie1@gmail.com>
Date: Sun, 18 Feb 2024 16:24:09 +0000
Subject: [PATCH] remove some comments

---
 src/generation.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/generation.py b/src/generation.py
index 106ddb2..49d7290 100644
--- a/src/generation.py
+++ b/src/generation.py
@@ -8,7 +8,7 @@
 
 from src.sample import sample
 from src.utils import print_rank_0
-from src.tokenizer import CharLevelTokenizer  # need to add a check for this type of tokenizer
+from src.tokenizer import CharLevelTokenizer
 
 
 class Generator:
@@ -33,7 +33,6 @@ def generate(
         stop_at_eos=True,
         max_seqlen=None,
     ):
-        # check dtype if self.tokenizer.eos is int
         if isinstance(self.tokenizer.eos, int):
             eos_token_ids = torch.LongTensor([self.tokenizer.eos]).to(device)
         else: