diff --git a/.gitignore b/.gitignore index 13b49bc..693761d 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,6 @@ geodjango/ tests-ui/screenshots/ static/ + + +static/ diff --git a/README.md b/README.md index b68f711..3291042 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ The folder `/fixtures` contains some test data, either as an SQL command to inse [`jq`](https://stedolan.github.io/jq/) is used for pretty-printing of the output. ```bash -# create dump after running test_data.sql: +# create dump after creating/harvesting test data: python manage.py dumpdata --exclude=auth --exclude=contenttypes | jq > fixtures/test_data.json # load: @@ -70,6 +70,9 @@ docker run --name optimetaPortalDB -p 5432:5432 -e POSTGRES_USER=optimeta -e POS python manage.py makemigrations python manage.py migrate +# create cache table +python manage.py createcachetable + # collect static files python manage.py collectstatic --noinput @@ -80,7 +83,7 @@ python manage.py runserver OPTIMAP_CACHE=dummy OPTIMAP_DEBUG=True python manage.py runserver ``` -Now open a browser at for the map and for the API. +Now open a browser at . ### Debug with VS Code @@ -115,6 +118,17 @@ Configuration for debugging with VS Code: Add `EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend` to the `.env` file to have emails printed to the console instead of sent via SMTP. +Alternatively, you can run a local STMP server with the following command and configuration: + +```bash +python -m smtpd -c DebuggingServer -n localhost:5587 +``` + +```env +OPTIMAP_EMAIL_HOST=localhost +OPTIMAP_EMAIL_PORT=5587 +``` + ### Create superusers/admin Superusers/admin can be created using the createsuperuser command: @@ -149,8 +163,6 @@ python -Wa manage.py test # running UI tests needs either compose configuration or a manage.py runserver in a seperate shell docker-compose up --build -# TODO insert test data - python -Wa manage.py test tests-ui ``` diff --git a/fixtures/test_data.json b/fixtures/test_data.json index 20bcdc7..8017e5b 100644 --- a/fixtures/test_data.json +++ b/fixtures/test_data.json @@ -10,7 +10,10 @@ "url": null, "geometry": "SRID=4326;GEOMETRYCOLLECTION (POINT (7.595730774920725 51.96944097112328), POLYGON ((7.599984296478425 51.984257653537384, 7.5715788777530975 51.97057414651397, 7.570122189613329 51.950602187631205, 7.580319006590855 51.93825551711683, 7.609054957094401 51.93035649564658, 7.659674869951374 51.942256350721436, 7.6833460522228165 51.968514669138415, 7.665137450475669 51.99229098076532, 7.626171042736502 51.98982421450293, 7.599984296478425 51.984257653537384)))", "creationDate": "2022-10-24T12:10:53.086Z", - "lastUpdate": "2022-10-24T12:10:53.086Z" + "lastUpdate": "2022-10-24T12:10:53.086Z", + "journal": "OPTIMAP Test Journal", + "timeperiod_startdate": "[\"2020-02-02\"]", + "timeperiod_enddate": "[\"2022-02-20\"]" } }, { @@ -24,7 +27,10 @@ "url": "http://paper.url/two", "geometry": "SRID=4326;GEOMETRYCOLLECTION (LINESTRING (9.754609563397707 52.36630414438588, 9.813062794192035 52.41569645624003, 10.141300167111496 52.36904961184797, 10.518997966087937 52.330597538337116, 10.838242534270051 52.311358956793185, 11.058566250338231 52.220550088821824, 11.535184901427073 52.15714903642342, 12.272594889905236 52.24258143981572, 12.618817872299417 52.35532056817789, 12.911084026269464 52.2976119913985, 13.144896949445211 52.50063147184562, 13.396695482095708 52.517051586549286))", "creationDate": "2022-10-24T12:10:53.086Z", - "lastUpdate": "2022-10-24T12:10:53.086Z" + "lastUpdate": "2022-10-24T12:10:53.086Z", + "journal": "OPTIMAP Test Journal", + "timeperiod_startdate": "[\"2010-01-01\"]", + "timeperiod_enddate": "[\"2012-12-12\"]" } } ] diff --git a/fixtures/test_data.sql b/fixtures/test_data.sql deleted file mode 100644 index 2d13325..0000000 --- a/fixtures/test_data.sql +++ /dev/null @@ -1,48 +0,0 @@ --- Draw geometry at https://geojson.io/#map=10.92/51.902/7.6702 --- possibly need to convert to GeometryCollection from FeatureCollection if we face that in GeoJSON the wild: https://gis.stackexchange.com/questions/177254/create-a-geosgeometry-from-a-featurecollection-in-geodangoINSERT INTO public.publications_publication VALUES - -INSERT INTO public.publications_publication VALUES - (1, 'The First Article', 'This is the first article. It is good in Münster.', '2010-10-10 10:10:10',NULL, 'https://service.tib.eu/optimeta/index.php/optimeta/article/view/8' , ST_GeomFromGeoJSON(' - { - "type": "GeometryCollection", - "geometries": [{ - "coordinates": [7.595730774920725,51.96944097112328], - "type": "Point", - "crs":{"type":"name","properties":{"name":"EPSG:4326"}} - }, { - "type": "Polygon", - "coordinates": [[ - [7.599984296478425,51.984257653537384], - [7.5715788777530975,51.97057414651397], - [7.570122189613329,51.950602187631205], - [7.580319006590855,51.93825551711683], - [7.609054957094401,51.93035649564658], - [7.659674869951374,51.942256350721436], - [7.6833460522228165,51.968514669138415], - [7.665137450475669,51.99229098076532], - [7.626171042736502,51.98982421450293], - [7.599984296478425,51.984257653537384] - ]] - }] - }'),'Journal of Optimal Geolocations', now(), now(), '{"2022-06-01"}','{"2022-06-08"}'), - (2, 'Paper Two', 'A second article. It is better; from Hanover to Berlin.', '2011-11-11 11:11:11','10.5555/12345678', NULL , ST_GeomFromGeoJSON(' - { - "type": "GeometryCollection", - "geometries": [{ - "type": "LineString", - "coordinates": [ - [9.754609563397707,52.36630414438588], - [9.813062794192035,52.41569645624003], - [10.141300167111496,52.36904961184797], - [10.518997966087937,52.330597538337116], - [10.838242534270051,52.311358956793185], - [11.058566250338231,52.220550088821824], - [11.535184901427073,52.15714903642342], - [12.272594889905236,52.24258143981572], - [12.618817872299417,52.35532056817789], - [12.911084026269464,52.2976119913985], - [13.144896949445211,52.50063147184562], - [13.396695482095708,52.517051586549286] - ] - }] - }'), 'Journal of Optimal Geolocations',now(), now(), '{"2022-02-01"}','{"2022-03-31"}') diff --git a/fly.io.md b/fly.io.md index 1f0d497..3e708bd 100644 --- a/fly.io.md +++ b/fly.io.md @@ -10,8 +10,8 @@ curl -L https://fly.io/install.sh | sh ## Create PostGIS -- https://community.fly.io/t/deploying-postgis/3530 -- https://fly.io/docs/reference/postgres-on-nomad/ +- +- ```bash flyctl postgres create @@ -45,9 +45,9 @@ fly launch --dockerfile Dockerfile Now say YES when asked if you want to create a Postgres DB. -> We recommend using the database_url(pip install dj-database-url) to parse the DATABASE_URL from os.environ['DATABASE_URL'] +> We recommend using the database_url (`pip install dj-database-url`) to parse the DATABASE_URL from os.environ['DATABASE_URL'] > -> For detailed documentation, see https://fly.dev/docs/django/ +> For detailed documentation, see Why not - changed configuration style to use `dj-database-uri`. @@ -142,9 +142,10 @@ and -## Update allowed hosts +## Update allowed hosts and configure CSRF - +- See for links and issue description around CSRF Add to `tly.toml`: @@ -154,7 +155,15 @@ Add to `tly.toml`: Then `flyctl deploy`. +## Connect to database + +```bash +fly proxy 15432:5432 -a optimap-db +``` + +Connect to database locally at port `15432`, e.g., with pgAdmin. + ## Future - Database backups, see -- Health check endpoint, see \ No newline at end of file +- Health check endpoint, see diff --git a/fly.toml b/fly.toml index 1807be8..6f38168 100644 --- a/fly.toml +++ b/fly.toml @@ -9,18 +9,22 @@ processes = [] dockerfile = "Dockerfile" [deploy] - release_command = "python manage.py migrate" + release_command = "sh release_command.sh" [env] - OPTIMAP_ALLOWED_HOST = "*" + OPTIMAP_ALLOWED_HOST = "optimap.science,optimap.fly.dev" PORT = "8000" OPTIMAP_DEBUG = false OPTIMAP_EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" - OPTIMAP_EMAIL_HOST= "smtp.ionos.com" - OPTIMAP_EMAIL_PORT = "587" + OPTIMAP_EMAIL_HOST= "smtp.ionos.de" + OPTIMAP_EMAIL_PORT_SMTP = "587" OPTIMAP_EMAIL_HOST_USER = "login@optimap.science" OPTIMAP_EMAIL_USE_TLS= true + OPTIMAP_EMAIL_HOST_IMAP = "imap.ionos.de" + OPTIMAP_EMAIL_PORT_IMAP = "993" + OPTIMAP_EMAIL_IMAP_SENT_FOLDER = "\"Gesendete Objekte\"" CSRF_TRUSTED_ORIGINS = "https://optimap.science" + OPTIMAP_LOGGING_CONSOLE_LEVEL = "DEBUG" [experimental] allowed_public_ports = [] diff --git a/optimetaPortal/.env.example b/optimetaPortal/.env.example index 359f212..071226f 100644 --- a/optimetaPortal/.env.example +++ b/optimetaPortal/.env.example @@ -11,7 +11,12 @@ OPTIMAP_DB_PORT=5432 OPTIMAP_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend OPTIMAP_EMAIL_HOST=smtp.gmail.com -OPTIMAP_EMAIL_PORT=587 +OPTIMAP_EMAIL_PORT_SMTP=587 OPTIMAP_EMAIL_HOST_USER=... OPTIMAP_EMAIL_HOST_PASSWORD=... +OPTIMAP_EMAIL_HOST_IMAP=imap.ionos.de +OPTIMAP_EMAIL_PORT_IMAP=993 OPTIMAP_EMAIL_USE_TLS=True +OPTIMAP_EMAIL_IMAP_SENT_FOLDER="" + +OPTIMAP_LOGGING_CONSOLE_LEVEL=INFO diff --git a/optimetaPortal/settings.py b/optimetaPortal/settings.py index acd9b82..ae3c24c 100644 --- a/optimetaPortal/settings.py +++ b/optimetaPortal/settings.py @@ -31,15 +31,17 @@ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env('SECRET_KEY', default='django-insecure-@(y&etxu!n5qkeyim8ineufd*c*0o20k6$q^$89md-i%qcdk57') +SECRET_KEY = env('SECRET_KEY', default='django-insecure') # SECURITY WARNING: don't run with debug turned on in production! DEBUG = env('OPTIMAP_DEBUG', default=True) -ALLOWED_HOSTS = env('OPTIMAP_ALLOWED_HOST', default=[]) +ALLOWED_HOSTS = [i.strip('[]') for i in env('OPTIMAP_ALLOWED_HOST', default='*').split(',')] OPTIMAP_SUPERUSER_EMAILS = [i.strip('[]') for i in env('OPTIMAP_SUPERUSER_EMAILS', default='').split(',')] +TEST_HARVESTING_ONLINE = env('OPTIMAP_TEST_HARVESTING_ONLINE', default=False) + ROOT_URLCONF = 'optimetaPortal.urls' AUTHENTICATION_BACKENDS = [ @@ -61,10 +63,13 @@ 'django_q', 'drf_spectacular', 'drf_spectacular_sidecar', + 'leaflet' # used in admin site ] REST_FRAMEWORK = { 'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema', + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', + 'PAGE_SIZE': 999, } # https://github.com/tfranzel/drf-spectacular @@ -139,10 +144,10 @@ } CACHES = { - # defaults to local-memory caching, see https://docs.djangoproject.com/en/4.1/topics/cache/#local-memory-caching + # defaults to database caching to persist across processes, see https://docs.djangoproject.com/en/4.1/topics/cache/#local-memory-caching 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'unique-snowflake', + 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', + 'LOCATION': 'cache', }, # use for development @@ -166,11 +171,15 @@ # for testing email sending EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' EMAIL_BACKEND = env('OPTIMAP_EMAIL_BACKEND', default='django.core.mail.backends.console.EmailBackend') -EMAIL_HOST = env('OPTIMAP_EMAIL_HOST', default='smtp.gmail.com') -EMAIL_PORT = env('OPTIMAP_EMAIL_PORT', default=587) -EMAIL_HOST_USER = env('OPTIMAP_EMAIL_HOST_USER', default=False) -EMAIL_HOST_PASSWORD = env('OPTIMAP_EMAIL_HOST_PASSWORD', default=False) -EMAIL_USE_TLS = env('OPTIMAP_EMAIL_USE_TLS', default=True) +EMAIL_HOST = env('OPTIMAP_EMAIL_HOST', default='optimeta.dev') +EMAIL_PORT = env('OPTIMAP_EMAIL_PORT_SMTP', default=587) +EMAIL_HOST_IMAP = env('OPTIMAP_EMAIL_HOST_IMAP', default='optimeta.imap') +EMAIL_PORT_IMAP = env('OPTIMAP_EMAIL_PORT_IMAP', default=993) +EMAIL_HOST_USER = env('OPTIMAP_EMAIL_HOST_USER', default='optimap@dev') +EMAIL_HOST_PASSWORD = env('OPTIMAP_EMAIL_HOST_PASSWORD', default='') +EMAIL_USE_TLS = env('OPTIMAP_EMAIL_USE_TLS', default=False) +EMAIL_USE_SSL = env('OPTIMAP_EMAIL_USE_SSL', default=False) +EMAIL_IMAP_SENT_FOLDER = env('OPTIMAP_EMAIL_IMAP_SENT_FOLDER', default='') MIDDLEWARE = [ 'django.middleware.cache.UpdateCacheMiddleware', @@ -273,7 +282,7 @@ }, 'handlers': { 'console': { - 'level': 'INFO', + 'level': 'DEBUG', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', 'formatter': 'simple' @@ -290,12 +299,18 @@ 'handlers': ['console', 'mail_admins'], 'level': 'INFO', }, + 'publications': { + 'handlers': ['console', 'mail_admins'], + 'level': env('OPTIMAP_LOGGING_CONSOLE_LEVEL', default='INFO'), + }, 'django.request': { 'handlers': ['mail_admins'], 'level': 'WARNING', 'propagate': False, - }, + } } } +CSRF_TRUSTED_ORIGINS = [i.strip('[]') for i in env('CSRF_TRUSTED_ORIGINS', default='https://localhost:8000').split(',')] + ADMINS = [('OPTIMAP', 'login@optimap.science')] diff --git a/publications/admin.py b/publications/admin.py index d685131..3bce120 100644 --- a/publications/admin.py +++ b/publications/admin.py @@ -1,14 +1,9 @@ from django.contrib import admin -"""Markers admin.""" - -from django.contrib.gis import admin - +from leaflet.admin import LeafletGeoAdmin from publications.models import Publication - @admin.register(Publication) -class PublicationAdmin(admin.OSMGeoAdmin): +class PublicationAdmin(LeafletGeoAdmin): """Publication Admin.""" list_display = ("title", "publicationDate", "creationDate", "lastUpdate") -# Register your models here. diff --git a/publications/api.py b/publications/api.py index a056e7a..7c940c1 100644 --- a/publications/api.py +++ b/publications/api.py @@ -2,9 +2,9 @@ from rest_framework import routers -from publications.viewsets import PublicationViewSet,SubscriptionViewset +from publications.viewsets import PublicationViewSet, SubscriptionViewset router = routers.DefaultRouter() router.register(r"publications", PublicationViewSet) -router.register(r"subscriptions",SubscriptionViewset,basename='Subscription') +router.register(r"subscriptions", SubscriptionViewset, basename='Subscription') urlpatterns = router.urls diff --git a/publications/migrations/0004_ojsservers_subscription_and_more.py b/publications/migrations/0004_ojsservers_subscription_and_more.py new file mode 100644 index 0000000..0705d85 --- /dev/null +++ b/publications/migrations/0004_ojsservers_subscription_and_more.py @@ -0,0 +1,79 @@ +# Generated by Django 4.0.5 on 2022-12-23 08:10 + +import django.contrib.gis.db.models.fields +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('publications', '0003_publication_journal'), + ] + + operations = [ + migrations.CreateModel( + name='OJSservers', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url_field', models.URLField()), + ('harvest_interval_minutes', models.IntegerField(default=4320)), + ('last_harvest', models.DateTimeField(auto_now_add=True, null=True)), + ], + ), + migrations.CreateModel( + name='Subscription', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('search_text', models.CharField(max_length=4096)), + ('timeperiod_startdate', models.DateField(null=True)), + ('timeperiod_enddate', models.DateField(null=True)), + ('search_area', django.contrib.gis.db.models.fields.GeometryCollectionField(blank=True, null=True, srid=4326)), + ('user_name', models.CharField(max_length=4096)), + ], + options={ + 'verbose_name': 'subscription', + 'ordering': ['user_name'], + }, + ), + migrations.AddField( + model_name='publication', + name='timeperiod_enddate', + field=django.contrib.postgres.fields.ArrayField(base_field=models.DateField(null=True), null=True, size=None), + ), + migrations.AddField( + model_name='publication', + name='timeperiod_startdate', + field=django.contrib.postgres.fields.ArrayField(base_field=models.DateField(null=True), null=True, size=None), + ), + migrations.AlterField( + model_name='publication', + name='abstract', + field=models.TextField(null=True), + ), + migrations.AlterField( + model_name='publication', + name='creationDate', + field=models.DateTimeField(auto_now_add=True, null=True), + ), + migrations.AlterField( + model_name='publication', + name='geometry', + field=django.contrib.gis.db.models.fields.GeometryCollectionField(blank=True, null=True, srid=4326, verbose_name='Publication geometry (Points, Lines, Polygons as GeoJSON)'), + ), + migrations.AlterField( + model_name='publication', + name='lastUpdate', + field=models.DateTimeField(auto_now=True, null=True), + ), + migrations.AlterField( + model_name='publication', + name='publicationDate', + field=models.DateField(null=True), + ), + migrations.AlterField( + model_name='publication', + name='title', + field=models.CharField(max_length=4096, null=True), + ), + ] diff --git a/publications/migrations/0005_alter_publication_abstract_and_more.py b/publications/migrations/0005_alter_publication_abstract_and_more.py new file mode 100644 index 0000000..aec7c9a --- /dev/null +++ b/publications/migrations/0005_alter_publication_abstract_and_more.py @@ -0,0 +1,59 @@ +# Generated by Django 4.0.5 on 2022-12-23 08:48 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('publications', '0004_ojsservers_subscription_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='publication', + name='abstract', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='publication', + name='creationDate', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='publication', + name='doi', + field=models.CharField(blank=True, max_length=1024, null=True), + ), + migrations.AlterField( + model_name='publication', + name='journal', + field=models.CharField(blank=True, max_length=1024, null=True), + ), + migrations.AlterField( + model_name='publication', + name='lastUpdate', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='publication', + name='timeperiod_enddate', + field=django.contrib.postgres.fields.ArrayField(base_field=models.DateField(null=True), blank=True, null=True, size=None), + ), + migrations.AlterField( + model_name='publication', + name='timeperiod_startdate', + field=django.contrib.postgres.fields.ArrayField(base_field=models.DateField(null=True), blank=True, null=True, size=None), + ), + migrations.AlterField( + model_name='publication', + name='title', + field=models.CharField(max_length=4096), + ), + migrations.AlterField( + model_name='publication', + name='url', + field=models.URLField(blank=True, max_length=1024, null=True), + ), + ] diff --git a/publications/migrations/0006_alter_publication_creationdate_and_more.py b/publications/migrations/0006_alter_publication_creationdate_and_more.py new file mode 100644 index 0000000..486334d --- /dev/null +++ b/publications/migrations/0006_alter_publication_creationdate_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.0.5 on 2023-01-27 08:59 + +import django.contrib.gis.db.models.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('publications', '0005_alter_publication_abstract_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='publication', + name='creationDate', + field=models.DateTimeField(auto_now_add=True), + ), + migrations.AlterField( + model_name='publication', + name='geometry', + field=django.contrib.gis.db.models.fields.GeometryCollectionField(blank=True, null=True, srid=4326, verbose_name='Publication geometry/ies'), + ), + ] diff --git a/publications/models.py b/publications/models.py index bc65cd6..918f211 100644 --- a/publications/models.py +++ b/publications/models.py @@ -3,18 +3,20 @@ class Publication(models.Model): - - title = models.CharField(max_length=4096,null =True) - abstract = models.TextField(null=True) + # required fields + title = models.CharField(max_length=4096) publicationDate = models.DateField(null=True) - doi = models.CharField(max_length=1024, null=True) - url = models.URLField(max_length=1024, null=True) - geometry = models.GeometryCollectionField(verbose_name='Publication geometry (Points, Lines, Polygons as GeoJSON)',srid = 4326, null=True, blank=True) - journal = models.CharField(max_length=1024, null=True) - creationDate = models.DateTimeField(auto_now_add=True,null=True) - lastUpdate = models.DateTimeField(auto_now=True,null=True) - timeperiod_startdate = ArrayField(models.DateField(null=True), null=True) - timeperiod_enddate = ArrayField(models.DateField(null=True), null=True) + creationDate = models.DateTimeField(auto_now_add=True) + lastUpdate = models.DateTimeField(auto_now=True) + + # possibly blank fields + abstract = models.TextField(null=True, blank=True) + doi = models.CharField(max_length=1024, null=True, blank=True) + url = models.URLField(max_length=1024, null=True, blank=True) + geometry = models.GeometryCollectionField(verbose_name='Publication geometry/ies', srid = 4326, null=True, blank=True) + journal = models.CharField(max_length=1024, null=True, blank=True) + timeperiod_startdate = ArrayField(models.DateField(null=True), null=True, blank=True) + timeperiod_enddate = ArrayField(models.DateField(null=True), null=True, blank=True) def __str__(self): """Return string representation.""" diff --git a/publications/serializers.py b/publications/serializers.py index ce54325..ce88ba2 100644 --- a/publications/serializers.py +++ b/publications/serializers.py @@ -11,7 +11,7 @@ class PublicationSerializer(serializers.GeoFeatureModelSerializer): class Meta: """publication serializer meta class.""" model = Publication - fields = ("title" ,"abstract", "publicationDate", "url", "doi","creationDate", "lastUpdate","timeperiod_startdate","timeperiod_enddate") + fields = ("id", "title" ,"abstract", "publicationDate", "url", "doi","creationDate", "lastUpdate","timeperiod_startdate","timeperiod_enddate") geo_field = "geometry" auto_bbox = True diff --git a/publications/static/css/main.css b/publications/static/css/main.css index f07afc0..60e0bb7 100644 --- a/publications/static/css/main.css +++ b/publications/static/css/main.css @@ -18,6 +18,13 @@ main { background-color: #158F9B; } +.tagline { + font-weight: 800; + font-size: 1.6em; + box-sizing: unset; + font-style: italic; +} + #sidebar { float: left; height: 100%; diff --git a/publications/static/css/rest_framework_tweaks.css b/publications/static/css/rest_framework_tweaks.css new file mode 100644 index 0000000..06536fe --- /dev/null +++ b/publications/static/css/rest_framework_tweaks.css @@ -0,0 +1,133 @@ +/* +https://github.com/encode/django-rest-framework/blob/master/rest_framework/static/rest_framework/css/bootstrap-tweaks.css +*/ + +ul.breadcrumb { + margin: 20px 0; + font-weight: bold; +} + +.breadcrumb li.active a { + color: #777; + padding: 0 10px; +} + +body a { + color: #158F9B; +} + +body a:hover { + color: #278189; +} + +.pagination>.disabled>a, +.pagination>.disabled>a:hover, +.pagination>.disabled>a:focus { + cursor: not-allowed; + pointer-events: none; +} + +.pager>.disabled>a, +.pager>.disabled>a:hover, +.pager>.disabled>a:focus { + pointer-events: none; +} + +.pager .next { + margin-left: 10px; +} + +html, body { + height: 100%; +} + +.wrapper { + position: relative; + top: 0; + left: 0; + padding-top: 60px; + margin: -60px 0; + min-height: 100%; +} + +.form-switcher { + margin-bottom: 0; +} + +.well { + -webkit-box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; +} + +.well .form-actions { + padding-bottom: 0; + margin-bottom: 0; +} + +.well form { + margin-bottom: 0; +} + +.nav-tabs { + border: 0; +} + +.nav-tabs>li { + float: right; +} + +.nav-tabs li a { + margin-right: 0; +} + +.nav-tabs>.active>a { + background: #F5F5F5; +} + +.nav-tabs>.active>a:hover { + background: #F5F5F5; +} + +.tabbable.first-tab-active .tab-content { + border-top-right-radius: 0; +} + +.page-header { + border-bottom: none; + padding-bottom: 0px; + margin: 0; +} + +/* custom general page styles */ +.hero-unit h1, .hero-unit h2 { + color: #A30000; +} + + +.request-info { + clear: both; +} + +.horizontal-checkbox label { + padding-top: 0; +} + +.horizontal-checkbox label { + padding-top: 0 !important; +} + +.horizontal-checkbox input { + float: left; + width: 20px; + margin-top: 3px; +} + +.modal-footer form { + margin-left: 5px; + margin-right: 5px; +} + +.pagination { + margin: 5px 0 10px 0; +} \ No newline at end of file diff --git a/publications/static/favicon.ico b/publications/static/favicon.ico new file mode 100644 index 0000000..1363dd2 Binary files /dev/null and b/publications/static/favicon.ico differ diff --git a/publications/static/js/main.js b/publications/static/js/main.js index 872a179..4fca609 100644 --- a/publications/static/js/main.js +++ b/publications/static/js/main.js @@ -1,5 +1,5 @@ const dataCopyright = " | Publication metadata license: CC-0"; -const publications_url = '/api/publications/'; +const publications_url = '/api/v1/publications.json?limit=999999'; async function initMap() { var map = L.map("map"); @@ -68,8 +68,9 @@ function publicationPopup(feature, layer) { async function load_publications() { response = await fetch(publications_url); - geojson = await response.json(); - return geojson; + body = await response.json(); + console.log('OPTIMAP retrieved ' + body.count + ' results.'); + return body.results; } // render publications after page is loaded diff --git a/publications/static/js/popper.js b/publications/static/js/popper.js new file mode 100644 index 0000000..1fb2256 --- /dev/null +++ b/publications/static/js/popper.js @@ -0,0 +1,6 @@ +/** + * @popperjs/core v2.11.6 - MIT License + */ + +!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?t(exports):"function"==typeof define&&define.amd?define(["exports"],t):t((e="undefined"!=typeof globalThis?globalThis:e||self).Popper={})}(this,(function(e){"use strict";function t(e){if(null==e)return window;if("[object Window]"!==e.toString()){var t=e.ownerDocument;return t&&t.defaultView||window}return e}function n(e){return e instanceof t(e).Element||e instanceof Element}function r(e){return e instanceof t(e).HTMLElement||e instanceof HTMLElement}function o(e){return"undefined"!=typeof ShadowRoot&&(e instanceof t(e).ShadowRoot||e instanceof ShadowRoot)}var i=Math.max,a=Math.min,s=Math.round;function f(){var e=navigator.userAgentData;return null!=e&&e.brands?e.brands.map((function(e){return e.brand+"/"+e.version})).join(" "):navigator.userAgent}function c(){return!/^((?!chrome|android).)*safari/i.test(f())}function p(e,o,i){void 0===o&&(o=!1),void 0===i&&(i=!1);var a=e.getBoundingClientRect(),f=1,p=1;o&&r(e)&&(f=e.offsetWidth>0&&s(a.width)/e.offsetWidth||1,p=e.offsetHeight>0&&s(a.height)/e.offsetHeight||1);var u=(n(e)?t(e):window).visualViewport,l=!c()&&i,d=(a.left+(l&&u?u.offsetLeft:0))/f,h=(a.top+(l&&u?u.offsetTop:0))/p,m=a.width/f,v=a.height/p;return{width:m,height:v,top:h,right:d+m,bottom:h+v,left:d,x:d,y:h}}function u(e){var n=t(e);return{scrollLeft:n.pageXOffset,scrollTop:n.pageYOffset}}function l(e){return e?(e.nodeName||"").toLowerCase():null}function d(e){return((n(e)?e.ownerDocument:e.document)||window.document).documentElement}function h(e){return p(d(e)).left+u(e).scrollLeft}function m(e){return t(e).getComputedStyle(e)}function v(e){var t=m(e),n=t.overflow,r=t.overflowX,o=t.overflowY;return/auto|scroll|overlay|hidden/.test(n+o+r)}function y(e,n,o){void 0===o&&(o=!1);var i,a,f=r(n),c=r(n)&&function(e){var t=e.getBoundingClientRect(),n=s(t.width)/e.offsetWidth||1,r=s(t.height)/e.offsetHeight||1;return 1!==n||1!==r}(n),m=d(n),y=p(e,c,o),g={scrollLeft:0,scrollTop:0},b={x:0,y:0};return(f||!f&&!o)&&(("body"!==l(n)||v(m))&&(g=(i=n)!==t(i)&&r(i)?{scrollLeft:(a=i).scrollLeft,scrollTop:a.scrollTop}:u(i)),r(n)?((b=p(n,!0)).x+=n.clientLeft,b.y+=n.clientTop):m&&(b.x=h(m))),{x:y.left+g.scrollLeft-b.x,y:y.top+g.scrollTop-b.y,width:y.width,height:y.height}}function g(e){var t=p(e),n=e.offsetWidth,r=e.offsetHeight;return Math.abs(t.width-n)<=1&&(n=t.width),Math.abs(t.height-r)<=1&&(r=t.height),{x:e.offsetLeft,y:e.offsetTop,width:n,height:r}}function b(e){return"html"===l(e)?e:e.assignedSlot||e.parentNode||(o(e)?e.host:null)||d(e)}function w(e){return["html","body","#document"].indexOf(l(e))>=0?e.ownerDocument.body:r(e)&&v(e)?e:w(b(e))}function x(e,n){var r;void 0===n&&(n=[]);var o=w(e),i=o===(null==(r=e.ownerDocument)?void 0:r.body),a=t(o),s=i?[a].concat(a.visualViewport||[],v(o)?o:[]):o,f=n.concat(s);return i?f:f.concat(x(b(s)))}function O(e){return["table","td","th"].indexOf(l(e))>=0}function j(e){return r(e)&&"fixed"!==m(e).position?e.offsetParent:null}function E(e){for(var n=t(e),i=j(e);i&&O(i)&&"static"===m(i).position;)i=j(i);return i&&("html"===l(i)||"body"===l(i)&&"static"===m(i).position)?n:i||function(e){var t=/firefox/i.test(f());if(/Trident/i.test(f())&&r(e)&&"fixed"===m(e).position)return null;var n=b(e);for(o(n)&&(n=n.host);r(n)&&["html","body"].indexOf(l(n))<0;){var i=m(n);if("none"!==i.transform||"none"!==i.perspective||"paint"===i.contain||-1!==["transform","perspective"].indexOf(i.willChange)||t&&"filter"===i.willChange||t&&i.filter&&"none"!==i.filter)return n;n=n.parentNode}return null}(e)||n}var D="top",A="bottom",L="right",P="left",M="auto",k=[D,A,L,P],W="start",B="end",H="viewport",T="popper",R=k.reduce((function(e,t){return e.concat([t+"-"+W,t+"-"+B])}),[]),S=[].concat(k,[M]).reduce((function(e,t){return e.concat([t,t+"-"+W,t+"-"+B])}),[]),V=["beforeRead","read","afterRead","beforeMain","main","afterMain","beforeWrite","write","afterWrite"];function q(e){var t=new Map,n=new Set,r=[];function o(e){n.add(e.name),[].concat(e.requires||[],e.requiresIfExists||[]).forEach((function(e){if(!n.has(e)){var r=t.get(e);r&&o(r)}})),r.push(e)}return e.forEach((function(e){t.set(e.name,e)})),e.forEach((function(e){n.has(e.name)||o(e)})),r}function C(e){return e.split("-")[0]}function N(e,t){var n=t.getRootNode&&t.getRootNode();if(e.contains(t))return!0;if(n&&o(n)){var r=t;do{if(r&&e.isSameNode(r))return!0;r=r.parentNode||r.host}while(r)}return!1}function I(e){return Object.assign({},e,{left:e.x,top:e.y,right:e.x+e.width,bottom:e.y+e.height})}function _(e,r,o){return r===H?I(function(e,n){var r=t(e),o=d(e),i=r.visualViewport,a=o.clientWidth,s=o.clientHeight,f=0,p=0;if(i){a=i.width,s=i.height;var u=c();(u||!u&&"fixed"===n)&&(f=i.offsetLeft,p=i.offsetTop)}return{width:a,height:s,x:f+h(e),y:p}}(e,o)):n(r)?function(e,t){var n=p(e,!1,"fixed"===t);return n.top=n.top+e.clientTop,n.left=n.left+e.clientLeft,n.bottom=n.top+e.clientHeight,n.right=n.left+e.clientWidth,n.width=e.clientWidth,n.height=e.clientHeight,n.x=n.left,n.y=n.top,n}(r,o):I(function(e){var t,n=d(e),r=u(e),o=null==(t=e.ownerDocument)?void 0:t.body,a=i(n.scrollWidth,n.clientWidth,o?o.scrollWidth:0,o?o.clientWidth:0),s=i(n.scrollHeight,n.clientHeight,o?o.scrollHeight:0,o?o.clientHeight:0),f=-r.scrollLeft+h(e),c=-r.scrollTop;return"rtl"===m(o||n).direction&&(f+=i(n.clientWidth,o?o.clientWidth:0)-a),{width:a,height:s,x:f,y:c}}(d(e)))}function F(e,t,o,s){var f="clippingParents"===t?function(e){var t=x(b(e)),o=["absolute","fixed"].indexOf(m(e).position)>=0&&r(e)?E(e):e;return n(o)?t.filter((function(e){return n(e)&&N(e,o)&&"body"!==l(e)})):[]}(e):[].concat(t),c=[].concat(f,[o]),p=c[0],u=c.reduce((function(t,n){var r=_(e,n,s);return t.top=i(r.top,t.top),t.right=a(r.right,t.right),t.bottom=a(r.bottom,t.bottom),t.left=i(r.left,t.left),t}),_(e,p,s));return u.width=u.right-u.left,u.height=u.bottom-u.top,u.x=u.left,u.y=u.top,u}function U(e){return e.split("-")[1]}function z(e){return["top","bottom"].indexOf(e)>=0?"x":"y"}function X(e){var t,n=e.reference,r=e.element,o=e.placement,i=o?C(o):null,a=o?U(o):null,s=n.x+n.width/2-r.width/2,f=n.y+n.height/2-r.height/2;switch(i){case D:t={x:s,y:n.y-r.height};break;case A:t={x:s,y:n.y+n.height};break;case L:t={x:n.x+n.width,y:f};break;case P:t={x:n.x-r.width,y:f};break;default:t={x:n.x,y:n.y}}var c=i?z(i):null;if(null!=c){var p="y"===c?"height":"width";switch(a){case W:t[c]=t[c]-(n[p]/2-r[p]/2);break;case B:t[c]=t[c]+(n[p]/2-r[p]/2)}}return t}function Y(e){return Object.assign({},{top:0,right:0,bottom:0,left:0},e)}function G(e,t){return t.reduce((function(t,n){return t[n]=e,t}),{})}function J(e,t){void 0===t&&(t={});var r=t,o=r.placement,i=void 0===o?e.placement:o,a=r.strategy,s=void 0===a?e.strategy:a,f=r.boundary,c=void 0===f?"clippingParents":f,u=r.rootBoundary,l=void 0===u?H:u,h=r.elementContext,m=void 0===h?T:h,v=r.altBoundary,y=void 0!==v&&v,g=r.padding,b=void 0===g?0:g,w=Y("number"!=typeof b?b:G(b,k)),x=m===T?"reference":T,O=e.rects.popper,j=e.elements[y?x:m],E=F(n(j)?j:j.contextElement||d(e.elements.popper),c,l,s),P=p(e.elements.reference),M=X({reference:P,element:O,strategy:"absolute",placement:i}),W=I(Object.assign({},O,M)),B=m===T?W:P,R={top:E.top-B.top+w.top,bottom:B.bottom-E.bottom+w.bottom,left:E.left-B.left+w.left,right:B.right-E.right+w.right},S=e.modifiersData.offset;if(m===T&&S){var V=S[i];Object.keys(R).forEach((function(e){var t=[L,A].indexOf(e)>=0?1:-1,n=[D,A].indexOf(e)>=0?"y":"x";R[e]+=V[n]*t}))}return R}var K={placement:"bottom",modifiers:[],strategy:"absolute"};function Q(){for(var e=arguments.length,t=new Array(e),n=0;n=0?-1:1,i="function"==typeof n?n(Object.assign({},t,{placement:e})):n,a=i[0],s=i[1];return a=a||0,s=(s||0)*o,[P,L].indexOf(r)>=0?{x:s,y:a}:{x:a,y:s}}(n,t.rects,i),e}),{}),s=a[t.placement],f=s.x,c=s.y;null!=t.modifiersData.popperOffsets&&(t.modifiersData.popperOffsets.x+=f,t.modifiersData.popperOffsets.y+=c),t.modifiersData[r]=a}},se={left:"right",right:"left",bottom:"top",top:"bottom"};function fe(e){return e.replace(/left|right|bottom|top/g,(function(e){return se[e]}))}var ce={start:"end",end:"start"};function pe(e){return e.replace(/start|end/g,(function(e){return ce[e]}))}function ue(e,t){void 0===t&&(t={});var n=t,r=n.placement,o=n.boundary,i=n.rootBoundary,a=n.padding,s=n.flipVariations,f=n.allowedAutoPlacements,c=void 0===f?S:f,p=U(r),u=p?s?R:R.filter((function(e){return U(e)===p})):k,l=u.filter((function(e){return c.indexOf(e)>=0}));0===l.length&&(l=u);var d=l.reduce((function(t,n){return t[n]=J(e,{placement:n,boundary:o,rootBoundary:i,padding:a})[C(n)],t}),{});return Object.keys(d).sort((function(e,t){return d[e]-d[t]}))}var le={name:"flip",enabled:!0,phase:"main",fn:function(e){var t=e.state,n=e.options,r=e.name;if(!t.modifiersData[r]._skip){for(var o=n.mainAxis,i=void 0===o||o,a=n.altAxis,s=void 0===a||a,f=n.fallbackPlacements,c=n.padding,p=n.boundary,u=n.rootBoundary,l=n.altBoundary,d=n.flipVariations,h=void 0===d||d,m=n.allowedAutoPlacements,v=t.options.placement,y=C(v),g=f||(y===v||!h?[fe(v)]:function(e){if(C(e)===M)return[];var t=fe(e);return[pe(e),t,pe(t)]}(v)),b=[v].concat(g).reduce((function(e,n){return e.concat(C(n)===M?ue(t,{placement:n,boundary:p,rootBoundary:u,padding:c,flipVariations:h,allowedAutoPlacements:m}):n)}),[]),w=t.rects.reference,x=t.rects.popper,O=new Map,j=!0,E=b[0],k=0;k=0,S=R?"width":"height",V=J(t,{placement:B,boundary:p,rootBoundary:u,altBoundary:l,padding:c}),q=R?T?L:P:T?A:D;w[S]>x[S]&&(q=fe(q));var N=fe(q),I=[];if(i&&I.push(V[H]<=0),s&&I.push(V[q]<=0,V[N]<=0),I.every((function(e){return e}))){E=B,j=!1;break}O.set(B,I)}if(j)for(var _=function(e){var t=b.find((function(t){var n=O.get(t);if(n)return n.slice(0,e).every((function(e){return e}))}));if(t)return E=t,"break"},F=h?3:1;F>0;F--){if("break"===_(F))break}t.placement!==E&&(t.modifiersData[r]._skip=!0,t.placement=E,t.reset=!0)}},requiresIfExists:["offset"],data:{_skip:!1}};function de(e,t,n){return i(e,a(t,n))}var he={name:"preventOverflow",enabled:!0,phase:"main",fn:function(e){var t=e.state,n=e.options,r=e.name,o=n.mainAxis,s=void 0===o||o,f=n.altAxis,c=void 0!==f&&f,p=n.boundary,u=n.rootBoundary,l=n.altBoundary,d=n.padding,h=n.tether,m=void 0===h||h,v=n.tetherOffset,y=void 0===v?0:v,b=J(t,{boundary:p,rootBoundary:u,padding:d,altBoundary:l}),w=C(t.placement),x=U(t.placement),O=!x,j=z(w),M="x"===j?"y":"x",k=t.modifiersData.popperOffsets,B=t.rects.reference,H=t.rects.popper,T="function"==typeof y?y(Object.assign({},t.rects,{placement:t.placement})):y,R="number"==typeof T?{mainAxis:T,altAxis:T}:Object.assign({mainAxis:0,altAxis:0},T),S=t.modifiersData.offset?t.modifiersData.offset[t.placement]:null,V={x:0,y:0};if(k){if(s){var q,N="y"===j?D:P,I="y"===j?A:L,_="y"===j?"height":"width",F=k[j],X=F+b[N],Y=F-b[I],G=m?-H[_]/2:0,K=x===W?B[_]:H[_],Q=x===W?-H[_]:-B[_],Z=t.elements.arrow,$=m&&Z?g(Z):{width:0,height:0},ee=t.modifiersData["arrow#persistent"]?t.modifiersData["arrow#persistent"].padding:{top:0,right:0,bottom:0,left:0},te=ee[N],ne=ee[I],re=de(0,B[_],$[_]),oe=O?B[_]/2-G-re-te-R.mainAxis:K-re-te-R.mainAxis,ie=O?-B[_]/2+G+re+ne+R.mainAxis:Q+re+ne+R.mainAxis,ae=t.elements.arrow&&E(t.elements.arrow),se=ae?"y"===j?ae.clientTop||0:ae.clientLeft||0:0,fe=null!=(q=null==S?void 0:S[j])?q:0,ce=F+ie-fe,pe=de(m?a(X,F+oe-fe-se):X,F,m?i(Y,ce):Y);k[j]=pe,V[j]=pe-F}if(c){var ue,le="x"===j?D:P,he="x"===j?A:L,me=k[M],ve="y"===M?"height":"width",ye=me+b[le],ge=me-b[he],be=-1!==[D,P].indexOf(w),we=null!=(ue=null==S?void 0:S[M])?ue:0,xe=be?ye:me-B[ve]-H[ve]-we+R.altAxis,Oe=be?me+B[ve]+H[ve]-we-R.altAxis:ge,je=m&&be?function(e,t,n){var r=de(e,t,n);return r>n?n:r}(xe,me,Oe):de(m?xe:ye,me,m?Oe:ge);k[M]=je,V[M]=je-me}t.modifiersData[r]=V}},requiresIfExists:["offset"]};var me={name:"arrow",enabled:!0,phase:"main",fn:function(e){var t,n=e.state,r=e.name,o=e.options,i=n.elements.arrow,a=n.modifiersData.popperOffsets,s=C(n.placement),f=z(s),c=[P,L].indexOf(s)>=0?"height":"width";if(i&&a){var p=function(e,t){return Y("number"!=typeof(e="function"==typeof e?e(Object.assign({},t.rects,{placement:t.placement})):e)?e:G(e,k))}(o.padding,n),u=g(i),l="y"===f?D:P,d="y"===f?A:L,h=n.rects.reference[c]+n.rects.reference[f]-a[f]-n.rects.popper[c],m=a[f]-n.rects.reference[f],v=E(i),y=v?"y"===f?v.clientHeight||0:v.clientWidth||0:0,b=h/2-m/2,w=p[l],x=y-u[c]-p[d],O=y/2-u[c]/2+b,j=de(w,O,x),M=f;n.modifiersData[r]=((t={})[M]=j,t.centerOffset=j-O,t)}},effect:function(e){var t=e.state,n=e.options.element,r=void 0===n?"[data-popper-arrow]":n;null!=r&&("string"!=typeof r||(r=t.elements.popper.querySelector(r)))&&N(t.elements.popper,r)&&(t.elements.arrow=r)},requires:["popperOffsets"],requiresIfExists:["preventOverflow"]};function ve(e,t,n){return void 0===n&&(n={x:0,y:0}),{top:e.top-t.height-n.y,right:e.right-t.width+n.x,bottom:e.bottom-t.height+n.y,left:e.left-t.width-n.x}}function ye(e){return[D,L,A,P].some((function(t){return e[t]>=0}))}var ge={name:"hide",enabled:!0,phase:"main",requiresIfExists:["preventOverflow"],fn:function(e){var t=e.state,n=e.name,r=t.rects.reference,o=t.rects.popper,i=t.modifiersData.preventOverflow,a=J(t,{elementContext:"reference"}),s=J(t,{altBoundary:!0}),f=ve(a,r),c=ve(s,o,i),p=ye(f),u=ye(c);t.modifiersData[n]={referenceClippingOffsets:f,popperEscapeOffsets:c,isReferenceHidden:p,hasPopperEscaped:u},t.attributes.popper=Object.assign({},t.attributes.popper,{"data-popper-reference-hidden":p,"data-popper-escaped":u})}},be=Z({defaultModifiers:[ee,te,oe,ie]}),we=[ee,te,oe,ie,ae,le,he,me,ge],xe=Z({defaultModifiers:we});e.applyStyles=ie,e.arrow=me,e.computeStyles=oe,e.createPopper=xe,e.createPopperLite=be,e.defaultModifiers=we,e.detectOverflow=J,e.eventListeners=ee,e.flip=le,e.hide=ge,e.offset=ae,e.popperGenerator=Z,e.popperOffsets=te,e.preventOverflow=he,Object.defineProperty(e,"__esModule",{value:!0})})); +//# sourceMappingURL=popper.min.js.map diff --git a/publications/tasks.py b/publications/tasks.py index a0c3e16..c27f94e 100644 --- a/publications/tasks.py +++ b/publications/tasks.py @@ -1,114 +1,106 @@ +import logging +logger = logging.getLogger(__name__) + from django_q.models import Schedule from publications.models import Publication from bs4 import BeautifulSoup import json import xml.dom.minidom -from django.contrib.gis.geos import GEOSGeometry +from django.contrib.gis.geos import GEOSGeometry import requests -def get_geom(url): - req= requests.get(url) - soup = BeautifulSoup(req.content, 'html.parser') - geom = parse_html(soup) - geom_object = None - if geom : - geom_data = geom["features"][0]["geometry"] - # preparing geometry data in accordance to geosAPI fields - type_geom= {'type': 'GeometryCollection'} - geom_content = {"geometries" : [geom_data]} - type_geom.update(geom_content) - geom_data_string= json.dumps(type_geom) - try : - geom_object = GEOSGeometry(geom_data_string) #GeometryCollection object - except : - print("Invalid Geometry") - - return geom_object - -def parse_html(content): - json_object = {} +def extract_geometry_from_html(content): for tag in content.find_all("meta"): if tag.get("name", None) == "DC.SpatialCoverage": data = tag.get("content", None) try: - json_object = json.loads(data) + geom = json.loads(data) + + geom_data = geom["features"][0]["geometry"] + # preparing geometry data in accordance to geosAPI fields + type_geom= {'type': 'GeometryCollection'} + geom_content = {"geometries" : [geom_data]} + type_geom.update(geom_content) + geom_data_string= json.dumps(type_geom) + try : + geom_object = GEOSGeometry(geom_data_string) # GeometryCollection object + logging.debug('Found geometry: %s', geom_object) + return geom_object + except : + print("Invalid Geometry") except ValueError as e: print("Not a valid GeoJSON") - return json_object - - -def get_timeperiod(url): - req= requests.get(url) - soup = BeautifulSoup(req.content, 'html.parser') - return extract_timeperiod_from_html(soup) def extract_timeperiod_from_html(content): - tp_start = [] - tp_end = [] + period = [None, None] for tag in content.find_all("meta"): - if tag.get("name", None) == "DC.temporal": + if tag.get("name", None) in ['DC.temporal', 'DC.PeriodOfTime']: data = tag.get("content", None) period = data.split("/") - period1 = period[0] - period2 = period[1] - tp_start.append(period1) - tp_end.append(period2) - - return tp_start,tp_end - + logging.debug('Found time period: %s', period) + break; + # returning arrays for array field in DB + return [period[0]], [period[1]] -def parse_xml(content): - +def parse_oai_xml_and_save_publications(content): DOMTree = xml.dom.minidom.parseString(content) collection = DOMTree.documentElement # pass DOMTree as argument - articles = collection.getElementsByTagName("dc:identifier") - articles_count_in_journal = len(articles) # number of articles in journal + articles = collection.getElementsByTagName("dc:identifier") + articles_count_in_journal = len(articles) for i in range(articles_count_in_journal): identifier = collection.getElementsByTagName("dc:identifier") identifier_value = identifier[i].firstChild.nodeValue if identifier_value.startswith('http'): - link_value = identifier_value - #get geometry from html - geom_object = get_geom(link_value) - #get Timeperiod from html - period = get_timeperiod(link_value) - period_start = period[0] - period_end = period[1] + + with requests.get(identifier_value) as response: + soup = BeautifulSoup(response.content, 'html.parser') + + geom_object = extract_geometry_from_html(soup) + period_start, period_end = extract_timeperiod_from_html(soup) + else: - link_value = None geom_object = None period_start = [] period_end = [] - + title = collection.getElementsByTagName("dc:title") if title: title_value = title[0].firstChild.nodeValue else : title_value = None - abstract = collection.getElementsByTagName("dc:description") + abstract = collection.getElementsByTagName("dc:description") if abstract: abstract_text = abstract[0].firstChild.nodeValue else: - abstract_text = None + abstract_text = None journal = collection.getElementsByTagName("dc:publisher") if journal: journal_value = journal[0].firstChild.nodeValue - else: + else: journal_value = None date = collection.getElementsByTagName("dc:date") if date: date_value = date[0].firstChild.nodeValue else: - date_value = None - publication = Publication(title = title_value,abstract = abstract_text,publicationDate = date_value, url = link_value , journal = journal_value, geometry = geom_object, timeperiod_startdate = period_start,timeperiod_enddate = period_end) + date_value = None + + publication = Publication( + title = title_value, + abstract = abstract_text, + publicationDate = date_value, + url = identifier_value, + journal = journal_value, + geometry = geom_object, + timeperiod_startdate = period_start, + timeperiod_enddate = period_end) publication.save() - + logger.info('Saved new publication for %s: %s', identifier_value, publication) -def harvest_data(url): +def harvest_oai_endpoint(url): try: - response = requests.get(url) - parse_xml(response.content) - except requests.exceptions.RequestException as e: + with requests.Session() as s: + response = s.get(url) + parse_oai_xml_and_save_publications(response.content) + except requests.exceptions.RequestException as e: print ("The requested URL is invalid or has bad connection.Please change the URL") - diff --git a/publications/templates/admin/base_site.html b/publications/templates/admin/base_site.html new file mode 100644 index 0000000..9f8a684 --- /dev/null +++ b/publications/templates/admin/base_site.html @@ -0,0 +1,17 @@ +{% extends "admin/base_site.html" %} +{% load static %} + +{% comment %} +See https://github.com/django/django/blob/main/django/contrib/admin/templates/admin/base_site.html +{% endcomment %} + +{% block extrahead %} + +{% endblock %} + +{% block branding %} +

OPTIMAP logo ADMIN

+{% if user.is_anonymous %} + {% include "admin/color_theme_toggle.html" %} +{% endif %} +{% endblock %} diff --git a/publications/templates/base.html b/publications/templates/base.html index abcf397..c451ef7 100644 --- a/publications/templates/base.html +++ b/publications/templates/base.html @@ -25,11 +25,13 @@