Merge pull request #494 from lmcmicu/dev_refactoring_jan_feb_2019

Refactor PURL system scripts

Makefile

@@ -2,6 +2,8 @@
 # 2015-11-06
 # James A. Overton <[email protected]>
 #
+# Last major modification: 2019-02-10, Michael Cuffaro <[email protected]>
+#
 # This file contains code for working with
 # Open Biomedical Ontoloiges (OBO)
 # Persistent Uniform Resource Locators (PURLs).
@@ -12,10 +14,8 @@
 # Required software:
 #
 # - [GNU Make](http://www.gnu.org/software/make/) to run this file
-# - [kwalify](http://www.kuwata-lab.com/kwalify/) for YAML validation
 # - [Python 3](https://www.python.org/downloads/) to run scripts
 # - [PyYAML](http://pyyaml.org/wiki/PyYAML) for translation to Apache
-# - [travis.rb](https://github.com/travis-ci/travis.rb) for Travis-CI
 
 
 ### Configuration
@@ -29,9 +29,6 @@
 # Defaults to the list of config/*.yml files.
 ONTOLOGY_IDS ?= $(patsubst config/%.yml,%,$(wildcard config/*.yml))
 
-# The GitHub owner/project
-PROJECT ?= OBOFoundry/purl.obolibrary.org
-
 # Local development server.
 DEVELOPMENT ?= localhost
 
@@ -53,219 +50,107 @@ SHELL := bash
 
 ### Basic Operations
 
-# Default goal: Remove generated files, validate config, and regenerate.
+# Default goal: Remove generated files and regenerate.
 .PHONY: all
-all: clean validate build
+all: clean build
 
-# Remove directories with generated files.
+# Remove directories with generated files and tests.
 .PHONY: clean
 clean:
 	rm -rf temp tests
 
 
-### Validate YAML Config
+### Build recipe for a single project.
 #
-# Use kwalify and the tools/config.schema.yml
-# to validate all YAML configuration files.
-# If any INVALID results are found, exit with an error.
-.PHONY: validate
-validate:
-	kwalify -f tools/config.schema.yml config/*.yml \
-	| awk '{print} /INVALID/ {status=1} END {exit status}'
-
-# Validate a single configuration file.
-.PHONY: validate-%
-validate-%:
-	kwalify -f tools/config.schema.yml config/$*.yml
+# Convert the YAML file of a single project to a .htaccess file and place it
+# in the temp/ directory.
+.PHONY: build-%
+build-%:
+	tools/translate_yaml.py --input_files config/$*.yml --output_dir temp
+	@echo "Built files in temp/$*"
 
 
-### Generate Apache Config
-#
-# Convert the YAML configuration files
-# to Apache .htaccess files with RedirectMatch directives.
-# There are three types:
-#
-# - base_redirects: when the project's base_url points to something
-# - product: for a project's main OWL file
-# - term: for a project's terms
-# - entries: PURLs under the project's base_url
+# Build recipe for all projects
 #
-# The first three are inserted into www/obo/.htaccess
-# while the last is in the project's www/obo/project/.htaccess
-#
-# These files are built in the `temp/` directory
-# then `temp/obo` replaces `www/obo` as the very last step
-# to keep Apache downtime to an absolute minimum.
-temp/obo temp/base_redirects temp/products temp/terms:
-	mkdir -p $@
+# Convert the YAML files of every project to .htaccess files and place them
+# in the www/obo directory.
 
-temp/base_redirects/%.htaccess: config/%.yml temp/base_redirects
-	tools/translate-base-redirects.py $< $@
-
-temp/products/%.htaccess: config/%.yml temp/products
-	tools/translate-products.py $< $@
-
-temp/terms/%.htaccess: config/%.yml temp/terms
-	tools/translate-terms.py $< $@
-
-# Generate temp/obo/foo/.htaccess file
-# and a symbolic link from the IDSPACE:
-# temp/obo/FOO -> temp/obo/foo
-# NOTE: The last line removes spurious links
-# on case insensitive file systems such as Mac OS X.
-temp/obo/%/.htaccess: config/%.yml
-	mkdir -p temp/obo/$*
-	tools/translate-entries.py $< $@
-	< $< \
-	grep '^idspace:' \
-	| sed 's/^idspace://' \
-	| tr -d ' ' \
-	| awk '{print "$* temp/obo/" $$0}' \
-	| xargs -t ln -s
-	rm -f temp/obo/$*/$*
-
-# Build temp files for a single project.
-.PHONY: build-%
-build-%: validate-% temp/obo/%/.htaccess temp/base_redirects/%.htaccess temp/products/%.htaccess temp/terms/%.htaccess
-	@echo "Built files in temp/$*"
+# Final output directory:
+www/obo/:
+	mkdir -p $@
 
+# When a new build is created, the old build's files are moved here, in a subdirectory
+# whose name is generated in a portable way using python (see the target-specific
+# variable BACKUP below).
 backup/:
 	mkdir $@
 
-# Get name of a dated-backup directory, in a portable way.
-BACKUP = backup/obo-$(shell python -c "import time,os;print(time.strftime('%Y%m%d-%H%M%S',time.gmtime(os.path.getmtime('www/obo'))))")
-
-# Convert all YAML configuration files to .htaccess
-# and move the special `obo` .htaccess file.
-# Generate .htaccess files for all YAML configuration files.
+# The main build target:
 .PHONY: build
-build: $(foreach o,$(ONTOLOGY_IDS),temp/obo/$o/.htaccess)
-build: $(foreach o,$(ONTOLOGY_IDS),temp/base_redirects/$o.htaccess)
-build: $(foreach o,$(ONTOLOGY_IDS),temp/products/$o.htaccess)
-build: $(foreach o,$(ONTOLOGY_IDS),temp/terms/$o.htaccess)
-build: | backup/
-	cat temp/obo/obo/.htaccess > temp/obo/.htaccess
-	echo '' >> temp/obo/.htaccess
-	echo '### Generated from project configuration files' >> temp/obo/.htaccess
-	echo '' >> temp/obo/.htaccess
-	cat temp/base_redirects/*.htaccess >> temp/obo/.htaccess
-	cat temp/products/*.htaccess >> temp/obo/.htaccess
-	cat temp/terms/*.htaccess >> temp/obo/.htaccess
-	rm -rf temp/obo/obo
-	rm -rf temp/obo/OBO
+build: BACKUP = backup/obo-$(shell python -c "import time,os;print(time.strftime('%Y%m%d-%H%M%S',time.gmtime(os.path.getmtime('www/obo'))))")
+build: | backup/ www/obo/
+	tools/translate_yaml.py --input_dir config --output_dir temp/obo
+	rm -rf temp/obo/obo temp/obo/OBO
 	-test -e www/obo && mv www/obo $(BACKUP)
 	mv temp/obo www/obo
+	rmdir temp
 
 
 ### Test Development Apache Config
 #
 # Make HTTP HEAD requests quickly against the DEVELOPMENT server
 # to ensure that redirects are working properly.
-tests/development:
-	mkdir -p $@
-
-# Run tests for a single YAML configuration file.
-# against the DEVELOPMENT server,
-# making requests every 0.01 seconds.
-tests/development/%.tsv: config/%.yml tests/development
-	tools/test.py --delay=0.01 $(DEVELOPMENT) $< $@
-
-# Run all tests against development and fail if any FAIL line is found.
+# Fail if any FAIL line is found in any of them.
 .PHONY: test
-test: $(foreach o,$(ONTOLOGY_IDS),tests/development/$o.tsv)
-	@cat tests/development/*.tsv \
-	| awk '/^FAIL/ {status=1; print} END {exit status}'
+test:
+	tools/test.py --delay=0.01 --output=tests/development --domain=$(DEVELOPMENT) config/*.yml
 
 
 ### Test Production Apache Config
 #
 # Make HTTP HEAD requests slowly against the PRODUCTION server
 # to ensure that redirects are working properly.
-tests/production:
-	mkdir -p $@
-
-# Run tests for a single YAML configuration file
-# against the PRODUCTION server,
-# making requests every 1 second.
-tests/production/%.tsv: config/%.yml tests/production
-	tools/test.py --delay=1 $(PRODUCTION) $< $@
-
-# Run all tests against production and fail if any FAIL line is found.
 .PHONY: test-production
-test-production: $(foreach o,$(ONTOLOGY_IDS),tests/production/$o.tsv)
-	@cat tests/production/*.tsv \
-	| awk '/^FAIL/ {status=1; print} END {exit status}'
+test-production:
+	tools/test.py --delay=1 --output=tests/production --domain=$(PRODUCTION) config/*.yml
 
 
 ### Test Tools
 #
 # Test our tools on files in examples/ directory.
-tests/examples:
-	mkdir -p $@
-
-tests/examples/%.yml: tools/examples/%.xml tools/examples/%.yml tests/examples
-	tools/migrate.py $* $< $@
-	diff tools/examples/$*.yml $@
-
-tests/examples/%.base_redirects.htaccess: tools/examples/%.yml tests/examples
-	tools/translate-base-redirects.py $< $@
-	diff tools/examples/$*.base_redirects.htaccess $@
-
-tests/examples/%.products.htaccess: tools/examples/%.yml tests/examples
-	tools/translate-products.py $< $@
-	diff tools/examples/$*.products.htaccess $@
-
-tests/examples/%.terms.htaccess: tools/examples/%.yml tests/examples
-	tools/translate-terms.py $< $@
-	diff tools/examples/$*.terms.htaccess $@
-
-tests/examples/%.htaccess: tools/examples/%.yml tests/examples
-	tools/translate-entries.py $< $@
-	diff tools/examples/$*.htaccess $@
+.PHONY: test-example1
+test-example1:
+	tools/migrate.py test1 tools/examples/test1/test1.xml tests/examples/test1/test1.yml
+	diff tools/examples/test1/test1.yml tests/examples/test1/test1.yml
+
+.PHONY: test-example2
+test-example2:
+	tools/translate_yaml.py --input_dir tools/examples/test2/ --output_dir tests/examples/test2/
+	diff tools/examples/test2/test2.htaccess tests/examples/test2/.htaccess
+	diff tools/examples/test2/obo/obo.htaccess tests/examples/test2/obo/.htaccess
+	diff tools/examples/test2/test2/test2.htaccess tests/examples/test2/test2/.htaccess	
 
 .PHONY: test-examples
-test-examples: tests/examples/test1.yml
-test-examples: tests/examples/test2.htaccess
-test-examples: tests/examples/test2.base_redirects.htaccess
-test-examples: tests/examples/test2.products.htaccess
-test-examples: tests/examples/test2.terms.htaccess
+test-examples: test-example1 test-example2
 
 
 ### Update Repository
 #
-# Check Travis-CI for the last build.
-# If it did not pass, then fail.
-# If it is the same as .current_build, then fail.
-# Otherwise replace .current_build,
-# pull from git, and run a new `make`.
+# Run the safe-update.py script which does the following:
+# - Check Travis-CI for the last build.
+# - If it did not pass, or if it is the same as the current build, then do nothing.
+# - Otherwise replace .current_build, pull from git, and run a new `make`.
 safe-update:
-	travis history --no-interactive \
-	--repo $(PROJECT) --branch master --limit 1 \
-	> .travis_build
-	@grep ' passed:   ' .travis_build
-	@echo 'Last build is green, but might not be new'
-	@diff .current_build .travis_build && exit 1 || exit 0
-	@echo 'New green build available'
-	@mv .travis_build .current_build
-	git pull
-	make
-
-
-### Migrate Configuration from PURL.org
+	tools/safe-update.py
+
+
+### Code style and lint checks for python source files.
 #
-# Given an ontology ID (usually lower-case),
-# fetch and translate a PURL.org XML file
-# into a YAML configuration file.
-# This should be a one-time migration.
-# Do not overwrite existing config file.
-PURL_XML = https://purl.org/admin/purl/?target=&seealso=&maintainers=&explicitmaintainers=&tombstone=false&p_id=
-
-.PHONY: migrate-%
-migrate-%:
-	@test ! -s config/$*.yml \
-	|| (echo 'Refusing to overwrite config/$*.yml'; exit 1)
-	mkdir -p migrations
-	test -s migrations/$*.xml \
-	|| curl --fail -o migrations/$*.xml "$(PURL_XML)/obo/$**"
-	mkdir -p config
-	tools/migrate.py $* migrations/$*.xml config/$*.yml
+# Note that `|| true` is appended to force make to ignore the exit code in both cases
+.PHONY: style
+style:
+	pep8 --max-line-length=100 --ignore E129,E126,E121,E111,E114 tools/*.py || true
+
+.PHONY: delint
+delint:
+	python3 -m pyflakes tools/*.py || true

README.md

@@ -202,6 +202,29 @@ or delete the VM with
 
 You can test against the production PURL server using `make test-production`. We only make one request per second, to avoid abusing the server, so this can take along time.
 
+### Optional: Sync VirtualBox Guest Additions
+
+If you keep your development VM for any length of time you may be presented with this message upon starting your VM:
+```
+==> default: A newer version of the box 'ubuntu/trusty64' is available! You currently
+==> default: have version '20190122.1.1'. The latest is version '20190206.0.0'. Run
+==> default: `vagrant box update` to update.
+```
+If you upgrade, then the next time you resume your box you may receive the warning:
+```
+[default] The guest additions on this VM do not match the install version of
+VirtualBox! This may cause things such as forwarded ports, shared
+folders, and more to not work properly. If any of those things fail on
+this machine, please update the guest additions and repackage the
+box.
+```
+
+To automatically sync with VirtualBox's Guest Additions at startup (and thus avoid this warning) you can install `vagrant-vbguest` like so:
+
+- `vagrant plugin install vagrant-vbguest` (in the tools directory on the host machine)
+
+Now, whenever you bring up your VM, it will check the version of the VM's guest additions and automatically bring them up to date whenever this is needed.
+
 
 ## Deployment
 

config/dpo.yml

@@ -10,6 +10,3 @@ products:
 - dpo.owl: https://raw.githubusercontent.com/FlyBase/flybase-controlled-vocabulary/master/releases/dpo.owl
 
 term_browser: ontobee
-example_terms:
-
-entries:

config/xao.yml

@@ -8,7 +8,6 @@ products:
 - xao.obo: http://ontologies.berkeleybop.org/xao.obo
 
 term_browser: ontobee
-example_terms:
 
 entries:
 - prefix: /tracker/