Auto detects when the text is latin1 or ascii and decode it as UTF-8 …

…so no strange chars are stored or used for the entry version comparisons

.gitignore

@@ -14,3 +14,4 @@ Pip*
 .env
 venv
 .idea
+*.ignore.py

diffengine/__init__.py

@@ -24,6 +24,7 @@
 import unicodedata
 
 from diffengine.sendgrid import SendgridHandler
+from diffengine.text import to_utf8
 from diffengine.twitter import TwitterHandler
 
 from exceptions.webdriver import UnknownWebdriverError
@@ -173,8 +174,7 @@ def get_latest(self):
         if resp.status_code != 200:
             logging.warn("Got %s when fetching %s", resp.status_code, self.url)
             return None
-
-        doc = readability.Document(resp.text)
+        doc = readability.Document(to_utf8(resp.text))
         title = doc.title()
         summary = doc.summary(html_partial=True)
         summary = bleach.clean(summary, tags=["p"], strip=True)

diffengine/text_builder.py → diffengine/text.py

@@ -58,3 +58,17 @@ def build_with_default_content(diff):
         text = text[0:225] + "…"
     text += " " + diff.url
     return text
+
+
+def to_utf8(text):
+    for encoding in ["latin1", "ascii"]:
+        try:
+            result = text.encode(encoding).decode("utf8", "strict")
+            break
+        except (UnicodeEncodeError, UnicodeDecodeError):
+            result = None
+
+    if result is None:
+        return text
+
+    return result

diffengine/twitter.py

@@ -3,7 +3,7 @@
 
 from datetime import datetime
 
-from diffengine.text_builder import build_text
+from diffengine.text import build_text
 from exceptions.twitter import (
     AlreadyTweetedError,
     TwitterConfigNotFoundError,

test_diffengine.py

@@ -29,7 +29,7 @@
     SendgridHandler,
     _fingerprint,
 )
-from diffengine.text_builder import build_text
+from diffengine.text import build_text, to_utf8
 from diffengine.utils import generate_config
 from exceptions.sendgrid import (
     SendgridConfigNotFoundError,
@@ -804,3 +804,16 @@ def test_lang_content_text(self):
         self.assertEqual(
             text, "change in the URL, the title and the summary\n%s" % diff.url
         )
+
+
+class EncodingTest(TestCase):
+    def test_utf8_do_nothingg(self):
+        text_utf8 = "Me preocupa más la parte futbolística"
+        result = to_utf8(text_utf8)
+        self.assertEquals(result, text_utf8)
+
+    def test_latin1_to_utf8(self):
+        text_latin = "Me preocupa más la parte futbolística"
+        text_utf8 = "Me preocupa más la parte futbolística"
+        result = to_utf8(text_latin)
+        self.assertEquals(result, text_utf8)