heroku run files

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 OurResearch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Procfile

@@ -0,0 +1,2 @@
+web: gunicorn views:app -w 2 --timeout 36000 --reload
+

README.md

@@ -1 +1,4 @@
 # openalex-guts
+==============
+
+The guts for computing data for OpenAlex. For more, see (https://openalex.org/)[https://openalex.org/].

app.py

@@ -0,0 +1,184 @@
+from flask import Flask
+from flask_sqlalchemy import SQLAlchemy
+from flask_compress import Compress
+from flask_debugtoolbar import DebugToolbarExtension
+
+from sqlalchemy import exc
+from sqlalchemy import event
+from sqlalchemy.pool import NullPool
+from sqlalchemy.pool import Pool
+
+import logging
+import sys
+import os
+import requests
+import json
+import random
+import warnings
+from urllib.parse import urlparse
+import psycopg2
+import psycopg2.extras # needed though you wouldn't guess it
+from psycopg2.pool import ThreadedConnectionPool
+from contextlib import contextmanager
+
+from util import safe_commit
+from util import elapsed
+from util import HTTPMethodOverrideMiddleware
+
+
+
+HEROKU_APP_NAME = "openalex-guts"
+
+# set up logging
+# see http://wiki.pylonshq.com/display/pylonscookbook/Alternative+logging+configuration
+logging.basicConfig(
+    stream=sys.stdout,
+    level=logging.DEBUG,
+    format='%(thread)d: %(message)s'  #tried process but it was always "6" on heroku
+)
+logger = logging.getLogger("oadoi")
+
+libraries_to_mum = [
+    "requests",
+    "urllib3",
+    "requests.packages.urllib3",
+    "requests_oauthlib",
+    "stripe",
+    "oauthlib",
+    "boto",
+    "newrelic",
+    "RateLimiter",
+    "paramiko",
+    "chardet",
+    "cryptography",
+    "psycopg2",
+    "snowflake"
+]
+
+
+for a_library in libraries_to_mum:
+    the_logger = logging.getLogger(a_library)
+    the_logger.setLevel(logging.WARNING)
+    the_logger.propagate = True
+    warnings.filterwarnings("ignore", category=UserWarning, module=a_library)
+
+# disable extra warnings
+requests.packages.urllib3.disable_warnings()
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+app = Flask(__name__)
+
+# database stuff
+app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = True  # as instructed, to suppress warning
+
+app.config['SQLALCHEMY_ECHO'] = (os.getenv("SQLALCHEMY_ECHO", False) == "True")
+# app.config['SQLALCHEMY_ECHO'] = True
+
+# app.config["SQLALCHEMY_DATABASE_URI"] = os.getenv("DATABASE_URL")  # don't use this though, default is unclear, use binds
+app.config["SQLALCHEMY_DATABASE_URI"] = os.getenv("DATABASE_URL_REDSHIFT")  # don't use this though, default is unclear, use binds
+app.config["SQLALCHEMY_BINDS"] = {
+    "unpaywall_db": os.getenv("DATABASE_URL_UNPAYWALL"),
+    "redshift_db": os.getenv("DATABASE_URL_REDSHIFT"),
+    "snowflake_db": os.getenv("DATABASE_URL_SNOWFLAKE"),
+    "paperbuzz_db": os.getenv("DATABASE_URL_PAPERBUZZ"),
+    "pubmed_db": os.getenv("DATABASE_URL_MEDOC")
+}
+
+# from http://stackoverflow.com/a/12417346/596939
+# class NullPoolSQLAlchemy(SQLAlchemy):
+#     def apply_driver_hacks(self, app, info, options):
+#         options['poolclass'] = NullPool
+#         return super(NullPoolSQLAlchemy, self).apply_driver_hacks(app, info, options)
+#
+# db = NullPoolSQLAlchemy(app, session_options={"autoflush": False})
+
+app.config["SQLALCHEMY_POOL_SIZE"] = 10
+db = SQLAlchemy(app, session_options={"autoflush": False, "autocommit": False})
+
+# do compression.  has to be above flask debug toolbar so it can override this.
+compress_json = os.getenv("COMPRESS_DEBUG", "True")=="True"
+
+
+# set up Flask-DebugToolbar
+if (os.getenv("FLASK_DEBUG", False) == "True"):
+    logger.info(u"Setting app.debug=True; Flask-DebugToolbar will display")
+    compress_json = False
+    app.debug = True
+    app.config['DEBUG'] = True
+    app.config["DEBUG_TB_INTERCEPT_REDIRECTS"] = False
+    app.config["SQLALCHEMY_RECORD_QUERIES"] = True
+    app.config["SECRET_KEY"] = os.getenv("SECRET_KEY")
+    toolbar = DebugToolbarExtension(app)
+
+# gzip responses
+Compress(app)
+app.config["COMPRESS_DEBUG"] = compress_json
+
+
+redshift_url = urlparse(os.getenv("DATABASE_URL_REDSHIFT"))
+app.config['postgreSQL_pool'] = ThreadedConnectionPool(2, 5,
+                                  database=redshift_url.path[1:],
+                                  user=redshift_url.username,
+                                  password=redshift_url.password,
+                                  host=redshift_url.hostname,
+                                  port=redshift_url.port)
+
+
+
+# import snowflake
+# class DictLowercaseCursor(snowflake.connector.DictCursor):
+#     def __init__(self, *args, **kwargs):  # it didn't work with or
+#         super().__init__(*args, **kwargs) # without these 2 lines
+#
+#     def execute(self, sql, args=None):
+#         sql =
+#         super().execute(sql, args)
+
+
+# @contextmanager
+def get_db_connection():
+    # conn = snowflake_engine.connect()
+
+    import snowflake.connector
+    from urllib.parse import urlparse
+    from urllib.parse import parse_qs
+    connection_parse = urlparse(os.getenv("DATABASE_URL_SNOWFLAKE"))
+    connection_query = parse_qs(connection_parse.query)
+
+    #create connection
+    conn = snowflake.connector.connect(
+        user=connection_query["user"][0],
+        password=connection_query["password"][0],
+        account=connection_parse.netloc,
+        warehouse=connection_query["warehouse"][0],
+        database=connection_query["db"][0],
+        schema=connection_query["schema"][0])
+    return conn
+
+    # try:
+    #     connection = app.config['postgreSQL_pool'].getconn()
+    #     connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+    #     connection.autocommit=True
+    #     # connection.readonly = True
+    #     yield connection
+    # finally:
+    #     app.config['postgreSQL_pool'].putconn(connection)
+
+# @contextmanager
+def get_db_cursor(commit=False):
+    import snowflake
+
+    conn = get_db_connection()
+    curs = conn.cursor(snowflake.connector.DictCursor)
+    return curs
+
+    # with get_db_connection() as connection:
+    #   cursor = connection.cursor(
+    #               cursor_factory=psycopg2.extras.RealDictCursor)
+    #   try:
+    #       yield cursor
+    #       if commit:
+    #           connection.commit()
+    #   finally:
+    #       cursor.close()
+    #       pass

requirements.txt

@@ -0,0 +1,43 @@
+blinker==1.4
+boto==2.49.0
+Brotli==1.0.9
+cached-property==1.5.2
+cachetools==4.2.2
+certifi==2021.5.30
+cffi==1.14.5
+chardet==3.0.4
+click==8.0.1
+cryptography==3.4.7
+Flask==2.0.1
+Flask-Compress==1.9.0
+Flask-DebugToolbar==0.11.0
+Flask-SQLAlchemy==2.5.1
+gunicorn==20.1.0
+heroku3==4.2.3
+humanfriendly==9.1
+idna==2.7
+itsdangerous==2.0.1
+Jinja2==3.0.1
+MarkupSafe==2.0.1
+MonthDelta==0.9.1
+packaging==20.9
+protobuf==3.17.2
+psycopg2-binary==2.8.6
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+pyOpenSSL==20.0.1
+pyparsing==2.4.7
+python-dateutil==2.6.0
+pytz==2021.1
+requests==2.20.0
+rsa==4.7.2
+simplejson==3.3.1
+six==1.16.0
+SQLAlchemy==1.4.17
+sqlalchemy-redshift==0.8.2
+unicodecsv==0.14.1
+Unidecode==1.2.0
+urllib3==1.24.3
+verboselogs==1.7
+Werkzeug==2.0.1

runtime.txt

@@ -0,0 +1 @@
+python-3.9.6