Merge pull request #290 from GSA/feature/scraper_improvements_8

Feature/scraper improvements 8

.cfignore

@@ -13,4 +13,5 @@ Dockerfile
 Dockerfile-test
 requirements.txt
 requirements-test.txt
-README.md
+README.md
+.venv*

.dockerignore

@@ -15,4 +15,8 @@ env/
 venv/
 .venv*
 crontab-test
-.vscode/
+.vscode/
+
+cf*
+bin*
+tests*

.gitignore

@@ -148,5 +148,8 @@ eda/attachment_texts/
 eda/labeled_fbo_docs/
 eda/models/
 
-# Smartie logs
-smartie*.log*
+# Smartie Logs:
+smartie*log*
+
+# Ruff
+*ruff*

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6.15-slim-buster
+FROM python:3.10-slim-buster
 
 ENV SUPERCRONIC_URL=https://github.com/albertcrowley/supercronic/releases/download/cloud-2/supercronic-linux-x86 \
     SUPERCRONIC=supercronic-linux-x86 \
@@ -42,8 +42,6 @@ RUN apt-get update && apt-get install -y \
     #clean up the apt cache
     && rm -rf /var/lib/apt/lists/*
 
-RUN pip install --upgrade pip
-RUN pip install -r requirements.txt --no-cache-dir
 
 RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
     && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \
@@ -53,11 +51,11 @@ RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key
     && unzip /tmp/chromedriver.zip chromedriver -d /usr/local/bin/
 
 ADD . .
-RUN python setup.py install
-COPY ./src /usr/local/lib/python3.6/site-packages/
+RUN pip install --upgrade pip && pip install -e .
 
 #see https://docs.cloudfoundry.org/devguide/deploy-apps/push-docker.html
 COPY ./conf/passwd /etc/passwd
+COPY ./conf /usr/local/conf
 
 ENTRYPOINT ["supercronic"]
 

README.md

@@ -59,18 +59,23 @@ To run the scan locally, do the following:
 ```bash
 cd path/to/this/locally/cloned/repo
 python3 -m venv env
-source venv/bin/activate
-pip install -r requirements.txt
-#now you can run the scan
-python main.py
+source env/bin/activate
+pip install -e ".[dev]"
+
+# Adding Configuration file to where the scraper can read it
+cd env
+ln -s <abs_path_to_repo>/conf conf
+
+# Now you can run the scan
+fbo_scraper
 ```
 
 #### Running the tests
 
 To run the tests the locally, set up the environment like before but instead run:
 
 ```bash
-python3 -W ignore -m unittest discover tests -p '*_test.py'
+py.test
 ```
 
 Several warnings and exceptions will print out. Those are by design as we're mocking HTTP requests in the unit testing.

alembic/dev/3df0b576624f_test.py

@@ -0,0 +1,44 @@
+"""Test
+
+Revision ID: 3df0b576624f
+Revises: 
+Create Date: 2023-08-03 11:11:26.143654
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.schema import Sequence, CreateSequence, DropSequence, Column
+
+# revision identifiers, used by Alembic.
+revision = '3df0b576624f'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### Alembic commands ###
+    op.create_primary_key('survey_responses_pkey', 'survey_responses', ['id'])
+    field_seq = Sequence('survey_responses_id_seq')
+    op.execute(CreateSequence(field_seq))
+    op.alter_column('survey_responses', 'id', server_default=field_seq.next_value())
+
+    op.add_column('Predictions', Column('active', sa.Boolean , default=True))
+
+    op.create_foreign_key('fk_notice_notice_type_id_notice_type', 'notice', 
+                          'notice_type', ['notice_type_id'], ["id"])
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### Alembic commands  ###
+    op.drop_constraint('survey_responses_pkey', 'survey_responses', type_='primary')
+    op.execute(DropSequence(Sequence('survey_responses_id_seq')))
+    op.alter_column('survey_responses', 'id', server_default=None)
+
+    op.drop_column('Predictions', 'active')
+
+    op.drop_constraint('fk_notice_notice_type_id_notice_type', 'notice', type_='foreignkey')
+    # ### end Alembic commands ###