Mccalluc/optional log exposure (#156)

* Add an old survey doc; link to poster * bump version * logs_path no longer the default * Support multiple file selection: debug app runs * Better names: "data" -> "files" * Hack to update list of running containers * Was getting 404 because we had enabled logs on the other demo, but not this one * Unused constant. * unused import * Update changelog [skip ci]
refinery-platform · Jul 24, 2018 · 9087def · 9087def
1 parent be13b24
commit 9087def
Show file tree

Hide file tree

Showing 15 changed files with 154 additions and 40 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Change Log
 
+## [v0.0.60](https://pypi.org/project/django-docker-engine/0.0.60/) (Jul 23, 2018)
+
+* Make log exposure optional.
+* In the demo, allow containers to be launched with multiple input files.
+
 ## [v0.0.59](https://pypi.org/project/django-docker-engine/0.0.59/) (Jul 10, 2018)
 
 * Expose logs in UI.

diff --git a/README-USERS.md b/README-USERS.md
@@ -144,7 +144,7 @@ True
 >>> (b'"GET / HTTP/1.1" 200' in api_logs) or api_logs
 True
 
-# ... or from the UI:
+# ... or from the UI, if `logs_path` was provided as a kwarg to Proxy:
 >>> ui_logs = requests.get(proxy_url + 'docker-logs').text
 >>> ('"GET / HTTP/1.1" 200' in ui_logs) or ui_logs
 True

diff --git a/README.md b/README.md
@@ -50,6 +50,6 @@ rankings based on heterogeneous attributes
 More information:
 - [for users of the library](https://github.com/refinery-platform/django_docker_engine/blob/master/README-USERS.md)
 - [for developers of the library](https://github.com/refinery-platform/django_docker_engine/blob/master/README-DEVS.md)
-- [background motivations and future directions](https://github.com/refinery-platform/django_docker_engine/blob/master/README-PROVENANCE.md)
+- [background reading](https://github.com/refinery-platform/django_docker_engine/blob/master/notes)
 - [API documentation](https://www.pydoc.io/pypi/django-docker-engine-0.0.57/)
-- [slides for BOSC 2018](https://docs.google.com/presentation/d/1Cc6XwVE5DhbnKRa07g7X1YkFihCzJn8j45LzhygDimY)
+- [BOSC 2018 poster](https://f1000research.com/posters/7-1078)
diff --git a/demo_path_routing_auth/forms.py b/demo_path_routing_auth/forms.py
@@ -3,9 +3,16 @@
 from .tools import tools
 
 
+class UnvalidatedMultipleChoiceField(forms.MultipleChoiceField):
+    # Django wants to know at startup what the possible values are,
+    # but the user could upload new files. Just skipping validation is fine.
+    def validate(self, value):
+        pass
+
+
 class LaunchForm(forms.Form):
     container_name = forms.CharField()
-    data = forms.CharField()  # Needs to be in definition to be readable.
+    files = UnvalidatedMultipleChoiceField()
     tool = forms.ChoiceField(
         widget=forms.Select,
         choices=tuple((k, k) for k in tools)

diff --git a/demo_path_routing_auth/index.html b/demo_path_routing_auth/index.html
@@ -69,7 +69,10 @@ <h2>Launch new container</h2>
       <form action="/launch/" method="post" target="_blank" id="launch">
         {% csrf_token %}
         <table>{{ launch_form.as_table }}</table>
-        <input type="submit" value="Launch" />
+        <input type="submit"
+               value="Launch"
+               onclick="setTimeout(function(){location.reload()}, 500);"/>
+        {# Reload original page to get updated containers list. A bit ugly. #}
       </form>
     </div>
 

diff --git a/demo_path_routing_auth/proxy_url_patterns.py b/demo_path_routing_auth/proxy_url_patterns.py
@@ -1,3 +1,3 @@
 from demo_path_routing_auth.proxy import AuthProxy
 
-urlpatterns = AuthProxy().url_patterns()
+urlpatterns = AuthProxy(logs_path='docker-logs').url_patterns()
diff --git a/demo_path_routing_auth/tools.py b/demo_path_routing_auth/tools.py
@@ -1,24 +1,27 @@
 tools = {
     'debugger': {
         'image': 'scottx611x/refinery-developer-vis-tool:v0.0.7',
-        'input': lambda url, prefix: {'file_relationships': [url]}
+        'input': lambda urls, prefix: {'file_relationships': urls}
     },
     'lineup': {
         'image': 'mccalluc/lineup_refinery:v0.0.8',
-        'input': lambda url, prefix: {'file_relationships': [url]}
+        'input': lambda urls, prefix: {'file_relationships': urls}
     },
     'higlass': {
         'image': 'scottx611x/refinery-higlass-docker:v0.3.2',
-        'input': lambda url, prefix: {
-            'node_info': {'fake-uuid': {'file_url': url}},
+        'input': lambda urls, prefix: {
+            'node_info': {
+                'fake-uuid-{}'.format(i): {'file_url': url}
+                for (i, url) in enumerate(urls)
+            },
             'extra_directories': ['/refinery-data/']
         }
     },
     'heatmap': {
         'image': 'mccalluc/heatmap_scatter_dash:v0.1.8',
-        'input': lambda url, prefix: {
+        'input': lambda urls, prefix: {
             'file_relationships': [
-                [url],  # Count data
+                urls,  # Count data
                 []  # Differential expression data
             ],
             'parameters': [],

diff --git a/demo_path_routing_auth/views.py b/demo_path_routing_auth/views.py
@@ -25,11 +25,11 @@
 def index(request):
     launch_form = LaunchForm()
     # TODO: Pass this info through the constructor
-    launch_form.fields['data'] = forms.ChoiceField(
-        widget=forms.Select,
+    launch_form.fields['files'] = forms.ChoiceField(
+        widget=forms.SelectMultiple,
         choices=((f, f) for f in os.listdir(UPLOAD_DIR) if f != '.gitignore')
     )
-    launch_form.initial['data'] = request.GET.get('uploaded')
+    launch_form.initial['files'] = [request.GET.get('uploaded')]
 
     context = {
         'container_names': [container.name for container in client.list()],
@@ -70,13 +70,15 @@ def launch(request):
         port = request.get_port()
     except AttributeError:  # Django 1.8.19
         port = request.get_host().replace('localhost:', '')
-    input_url = 'http://{}:{}/upload/{}'.format(
-        hostname(), port, post['data'])
+    input_urls = [
+        'http://{}:{}/upload/{}'.format(hostname(), port, file)
+        for file in post['files']
+    ]
     tool_spec = tools[post['tool']]
 
     container_name = post['container_name']
     container_path = '/docker/{}/'.format(container_name)
-    input_data = tool_spec['input'](input_url, container_path)
+    input_data = tool_spec['input'](input_urls, container_path)
 
     if post.get('show_input'):
         return HttpResponse(json.dumps(input_data), content_type='application/json')

diff --git a/demo_path_routing_no_auth/proxy_url_patterns.py b/demo_path_routing_no_auth/proxy_url_patterns.py
@@ -1,3 +1,3 @@
 from django_docker_engine.proxy import Proxy
 
-urlpatterns = Proxy().url_patterns()
+urlpatterns = Proxy(logs_path='docker-logs').url_patterns()
diff --git a/django_docker_engine/VERSION.txt b/django_docker_engine/VERSION.txt
@@ -1 +1 @@
-0.0.60
+0.0.61
diff --git a/django_docker_engine/proxy.py b/django_docker_engine/proxy.py
@@ -48,15 +48,16 @@ class Proxy():
     def __init__(self,
                  historian=NullHistorian(),
                  please_wait_title='Please wait',
-                 please_wait_body_html='<h1>Please wait</h1>'
-                 '<a href="docker-logs">View logs</a>',
-                 csrf_exempt=True):
+                 please_wait_body_html='<h1>Please wait</h1>',
+                 csrf_exempt=True,
+                 logs_path=None):
         self.historian = historian
         self.csrf_exempt = csrf_exempt
         self.content = self._render({
             'title': please_wait_title,
             'body_html': please_wait_body_html
         })
+        self.logs_path = logs_path
 
     def _render(self, context):
         template_path = os.path.join(
@@ -74,18 +75,21 @@ def _render(self, context):
         return template.render(context)
 
     def url_patterns(self):
-        return [
-            url(
-                r'^(?P<container_name>[^/]*)/docker-logs$',
-                csrf_exempt_decorator(self._logs_view) if self.csrf_exempt
-                else self._logs_view
-            ),
-            url(
-                r'^(?P<container_name>[^/]*)/(?P<url>.*)$',
-                csrf_exempt_decorator(self._proxy_view) if self.csrf_exempt
-                else self._proxy_view
-            )
-        ]
+        proxy_url = url(
+            r'^(?P<container_name>[^/]*)/(?P<url>.*)$',
+            csrf_exempt_decorator(self._proxy_view) if self.csrf_exempt
+            else self._proxy_view
+        )
+        if self.logs_path:
+            return [
+                url(
+                    r'^(?P<container_name>[^/]*)/{}$'.format(self.logs_path),
+                    csrf_exempt_decorator(self._logs_view) if self.csrf_exempt
+                    else self._logs_view
+                ),
+                proxy_url
+            ]
+        return [proxy_url]
 
     def _internal_proxy_view(self, request, container_url, path_url):
         # Any dependencies on the 3rd party proxy should be contained here.

diff --git a/README-PROVENANCE.md → notes/provenance.md b/README-PROVENANCE.md → notes/provenance.md
diff --git a/notes/survey.md b/notes/survey.md
@@ -0,0 +1,89 @@
+# User input and containerized vidualizations
+
+How do existing dockerized visualizations get their backing data? Are there idioms we can expect outside authors to follow?
+
+Conclusions:
+- Docker documentation is often an afterthought, though there are exceptions. Often just a copy of github docs.
+- Surprised by the number of desktop applications.
+- Mounting a /data volume is the usual approach to getting files in there, though we also see STDIN, sockets, and providing credentials for an API.
+
+
+## Best Webapp Examples:
+
+https://hub.docker.com/r/gehlenborglab/higlass/
+	Data to be visualized must be in a mounted volume, and after boot ingest_tileset must be run. (I could imagine every file in the mounted volume being ingested on start up, with a mapping from filenames to uids, with the software also generating a plausible viewconf for a given set of files.)
+
+https://hub.docker.com/r/vanessa/cogatvoxel/
+	Visualizing Brain anatomy? Data is baked in?
+
+https://hub.docker.com/r/tddv/superset/
+	OLAP: Guess you provide connection info for DB somehow?
+
+
+
+## Other Webapps:
+
+https://hub.docker.com/r/centurylink/imagelayers-ui/ -> http://imagelayers.io
+	Tool for examining shared dependencies between images. Seems broken right now, but I think the docker tag is passed as a URL parameter, and then it hits the dockerhub API.
+
+https://hub.docker.com/r/ouven/akka-visual-mailbox-visualization/
+	Visualize the state of a message passing system. (Reminds me of Erlang?) Web app, but another port is opened up where communication with the monitored system takes place.
+
+https://hub.docker.com/r/jldowns/vatic-docker/
+	Manage mechanical turk video annotation tasks. Mounted volumes for input and output files.
+
+https://hub.docker.com/r/inistcnrs/ezvis/
+	Visualization of natural language corpus. Example data hard coded?
+
+https://hub.docker.com/r/centurylink/image-graph/
+	Visualizing docker images. Mount the socket file to get information about the local Docker. Can either produce as output a PNG, or run as a webapp.
+
+
+
+## Desktop:
+
+https://hub.docker.com/r/nitnelave/deepvis/
+	Visualizing deep learning on image data. Desktop app rather than web app: I guess it works with X Window client-server, or something like that?
+
+https://hub.docker.com/r/jupedsim/jpsvis/
+	Pedestrian behavior simulator. Visualize output trajectory files.
+
+https://hub.docker.com/r/gtrafimenkov/logstalgia/
+	Server log analysis. Mount the directory containing the log.
+
+https://hub.docker.com/r/gtrafimenkov/codeswarm/
+	Visualize software project history. Mount the repo to visualize. Output is non-interactive video, I think?
+
+https://hub.docker.com/r/jonasrauber/c2s/
+	Neuroscience signal processing and visualization. Mount a data directory
+
+
+
+## Other / Unclear:
+
+
+https://hub.docker.com/r/funkwerk/compose_plantuml/
+	Generate UML from docker-compose specs. STDIN / STDOUT
+
+https://hub.docker.com/r/3dechem/largevis/
+	Sci-vis: high dimensional clustering. Mount a volume with the data on start. Produces output file.
+
+https://hub.docker.com/r/stennie/ubuntu-mtools/
+
+https://hub.docker.com/r/sevenbridges/maftools/
+	An R package? Docker container installs R and libraries.
+
+https://hub.docker.com/r/shopuz/alveo-visualization/
+	Unclear / Broken? I think it relies on an outside API to provide metadata on a corpus of human language data.
+
+https://hub.docker.com/r/ecomobi/visualization/
+	Stub? The company is something like a web ad network for southeast Asia...
+
+https://hub.docker.com/r/mathiask/weatherdb-visualization/
+	Stub?
+
+https://hub.docker.com/r/stefan125/sofia-service-osgi-visualization-ui/
+	Stub?
+
+https://hub.docker.com/r/kennethzfeng/pep-visualize/
+	Not sure where the data comes from.
diff --git a/tests/test_demo.py b/tests/test_demo.py
@@ -25,11 +25,11 @@ def test_home(self):
         context = response.context
 
         fields = context['launch_form'].fields
-        self.assertEqual(['container_name', 'data', 'show_input', 'tool'],
+        self.assertEqual(['container_name', 'files', 'show_input', 'tool'],
                          sorted(set(fields.keys())))
         # More wrapping for older pythons / older djangos.
 
-        self.assertIn(('3x3.csv', '3x3.csv'), fields['data'].choices)
+        self.assertIn(('3x3.csv', '3x3.csv'), fields['files'].choices)
         # Locally, you may also have data choices which are not checked in.
 
         self.assertEquals(
@@ -41,7 +41,7 @@ def test_home(self):
 
         self.assertEqual(
             context['launch_form'].initial,
-            {'data': '3x3.csv'})
+            {'files': ['3x3.csv']})
 
         content = response.content.decode('utf-8')
         self.assertIn('<option value="debugger">debugger</option>', content)
@@ -58,7 +58,7 @@ def test_lauch_post(self):
         with patch.object(DockerClientRunWrapper,
                           'run') as mock_run:
             response = self.client.post('/launch/',
-                                        {'data': 'fake-data',
+                                        {'files': ['fake-data'],
                                          'tool': 'debugger',
                                          'container_name': 'fake-name'},
                                         follow=True)

diff --git a/tests/test_proxy.py b/tests/test_proxy.py
@@ -48,7 +48,8 @@ def test_proxy_please_wait(self):
         proxy = Proxy(
             historian=historian,
             please_wait_title='<' + title_text + '>',
-            please_wait_body_html=body_html
+            please_wait_body_html=body_html,
+            logs_path='docker-logs'
         )
         urlpatterns = proxy.url_patterns()
         self.assertEqual(len(urlpatterns), 2)