Database: adapt models to be able to save scraped/parsed intermediate…

… data
PLNech · Sep 28, 2015 · ac583b0 · ac583b0
1 parent 7a00709
commit ac583b0
Show file tree

Hide file tree

Showing 7 changed files with 21 additions and 75 deletions.
diff --git a/menus/fixtures/initial_data.json b/menus/fixtures/initial_data.json
diff --git a/menus/models.py b/menus/models.py
@@ -73,14 +73,16 @@ def likes(self, dish):
 
 
 class RecipeToIngredient(models.Model):
-    #FIXME SHOULD BE UNIQUE FOR A RECIPE/INGREDIENT PAIR!
     recipe = models.ForeignKey('Recipe')
     ingredient = models.ForeignKey('Ingredient')
     quantity = models.FloatField(blank=True, null=True, default=None)
     unit = models.CharField(max_length=128, blank=True, null=True, default=None)
+    scraped_text = models.CharField(max_length=256, blank=True, null=True, default=None)
+    parsed_name = models.CharField(max_length=256, blank=True, null=True, default=None)
 
 
 class Recipe(models.Model):
+    dom = models.TextField(blank=True, null=True)
     name = models.CharField(max_length=128)
     picture = StdImageField(upload_to='media/images/recipe')
     prep_time = models.IntegerField()
@@ -113,7 +115,7 @@ def __str__(self):
 
 
 class Ingredient(models.Model):
-    name = models.CharField(max_length=32)
+    name = models.CharField(max_length=128)
     # country =  #  FIXME : use it for local menu generation
     # season?
     family = models.ForeignKey('IngredientFamily')
@@ -134,9 +136,8 @@ class IngredientNutriment(models.Model):
 
 
 class IngredientFamily(models.Model):
-    name = models.CharField(max_length=32)
+    name = models.CharField(max_length=64)
     picture = StdImageField(upload_to='media/images/ingredient_family')
-
     ingredients = models.ManyToManyField('Ingredient')
 
     # TODO (addition by Kevin) : seems necessary for hierarchy (to discuss)

diff --git a/testing/models.py b/testing/models.py
@@ -1,13 +1 @@
 from django.db import models
-
-class Ingredient(models.Model):
-    name = models.CharField(max_length=256)
-    category = models.CharField(max_length=512)
-    nutriments = models.ManyToManyField('Nutriment')
-
-class Nutriment(models.Model):
-    name = models.CharField(max_length=256)
-
-class Comment(models.Model):
-    url = models.URLField()
-    text = models.TextField()
diff --git a/testing/recipe_engine/db_link.py b/testing/recipe_engine/db_link.py
@@ -84,6 +84,7 @@ def save_recipe(recipe):
         "assez cher": 2
     }
     r = Recipe()
+    r.dom = recipe.dom
     r.name = recipe.title
     r.picture = recipe.picture_url
     r.prep_time = recipe.preptime
@@ -107,7 +108,9 @@ def save_recipe(recipe):
                 recipe=r,
                 ingredient=matched,
                 quantity=parsed.quantity,
-                unit=parsed.unit
+                unit=parsed.unit,
+                scraped_text=parsed.text,
+                parsed_name=parsed.name,
             )
             rtoi.save()
 
diff --git a/testing/recipe_engine/scraper.py b/testing/recipe_engine/scraper.py
@@ -34,6 +34,7 @@ def __init__(self, url):
         response = requests.get(url)
         soup = BeautifulSoup(response.text, "html.parser")
 
+        self.dom = response.text
         self.title = soup.find("span", "fn").string if soup.find("span", "fn") else None
         self.url = response.url
         self.picture_url = soup.find("img", { "class" : "photo"})['src'] if soup.find("img", { "class" : "photo"}) else None

diff --git a/testing/templates/recipes.html b/testing/templates/recipes.html
@@ -120,7 +120,7 @@
           <tbody>
           <tr>
               <th>Parsed</th>
-              <th>Matched</th>
+              <th>Matched (Ciqual)</th>
           </tr>
           {% for parsed, matched in matched_ingredients.items %}
           <tr>
@@ -138,52 +138,17 @@
   </div>
 
 
-    <div class="row">
-        <div class="col-sm-12">
-            <div class="col-sm-1 pull-right">
-                <form class="form-horizontal" action="{% url 'testing.views.recipes' %}" method="POST">
-                    <input type="hidden" name="recipe_url" value="{{ recipe.url }}"/>
-                    <input type="hidden" name="save" value="go"/>
-                    {% csrf_token %}
-                    <button class="btn btn-default pull-right" type="submit">Save</button>
-                </form>
-            </div>
-        </div>
-    </div>
-
-
-  <ul class="nav nav-tabs col-sm-12">
-    <li class="active"> <a href="#new-comment" data-toggle="tab">New comment</a> </li>
-    <li> <a href="#comments" data-toggle="tab">Comments</a> </li>
-  </ul>
-
-  <div class="row tab-content">
-    <div class="col-sm-6 tab-pane active" id="new-comment">
-      <form class="form-horizontal" action="{% url 'testing.views.recipes' %}" method="POST">
-        <label for="comment-input" class="control-label">Comment about this page:</label>
-        <textarea type="text" id="comment-input" name="new_comment" class="form-control" rows="4"></textarea>
-        <input type="hidden" name="com_url" value="{{ recipe.url }}"/>
-        {% csrf_token %}
-        <button class="btn btn-default col-sm-1" type="submit">Save</button>
-      </form>
-    </div>
-
-    <div class="row tab-pane" id="comments">
-      <div class="col-sm-6">
-      {% for com in comments %}
-        <div class="well row">
-            <p class="text-left col-sm-9"><a href="{{ com.url }}">{{ com.url }}</a><br/>{{ com.text }}</p>
-            <div class="text-right">
-            <form action="{% url 'testing.views.recipes' %}" method="POST">
-              <input type="hidden" name="recipe_url" value="{{ com.url }}"/>
-              {% csrf_token %}
-              <button class="btn btn-default" type="submit">Load url</button>
-            </form>
-            </div>
-        </div>
-      {% endfor %}
+  <div class="row">
+      <div class="col-sm-12">
+          <div class="col-sm-1 pull-right">
+              <form class="form-horizontal" action="{% url 'testing.views.recipes' %}" method="POST">
+                  <input type="hidden" name="recipe_url" value="{{ recipe.url }}"/>
+                  <input type="hidden" name="save" value="go"/>
+                  {% csrf_token %}
+                  <button class="btn btn-default pull-right" type="submit">Save</button>
+              </form>
+          </div>
       </div>
-    </div>
   </div>
 
 </div>

diff --git a/testing/views.py b/testing/views.py
@@ -1,7 +1,6 @@
 from django.shortcuts import render
 from django.conf import settings
 
-from testing.models import Comment
 from testing.recipe_engine.scraper import random_recipe, Recipe
 from testing.recipe_engine.db_link import get_matching_ingredients, save_recipe
 
@@ -19,19 +18,9 @@ def recipes(request):
     screenshot = settings.MEDIA_ROOT + '/screen.jpg'
     matched_ingredients = get_matching_ingredients(recipe.ingredients)
 
-    com_saved = False
-    if 'new_comment' in request.POST and request.POST['new_comment']:
-        if 'com_url' in request.POST and request.POST['com_url']:
-            com = Comment(url=request.POST['com_url'], text=request.POST['new_comment'])
-            com.save()
-            com_saved = True
-    comments = Comment.objects.all()
-
     return render(request, 'recipes.html', {
         'recipe': recipe,
         'screenshot': screenshot,
         'matched_ingredients': matched_ingredients,
-        'com_saved': com_saved,
-        'comments': comments
     })