Skip to content

Commit 9346d1f

Browse files
CastronovaCastronova
authored
Merge pull request #133 from hydroshare/develop
 HydroShare-JupyterHub v1.3
2 parents 3b13fa5 + 0747f9d commit 9346d1f

File tree

97 files changed

+6343
-1042
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

97 files changed

+6343
-1042
lines changed

‎.gitignore

+3-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
jupyterhub/env
2-
jupyterhub/jupyterhub.log
3-
jupyterhub/jupyterhub.sqlite
4-
jupyterhub/jupyterhub_cookie_secret
5-
jupyterhub/test
1+
**/secrets
2+
**/*env
3+
*.swp
64
*.pyc

‎build/celeryworker/Dockerfile

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
FROM python:2.7
2+
ADD requirements.txt /app/requirements.txt
3+
4+
#ADD ./celeryworker/ /app/utilities
5+
ADD ./celeryworker/ /app
6+
#RUN touch /app/utilities/__init__.py
7+
WORKDIR /app/
8+
RUN pip install -r requirements.txt
9+
10+
# add docker-cli
11+
RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-17.09.0-ce.tgz \
12+
&& tar -xf docker-17.09.0-ce.tgz \
13+
&& cp docker/docker /usr/bin/docker
14+
15+
ENTRYPOINT celery -A celeryworker worker --loglevel=info

‎build/celeryworker/build.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
docker build -t cuahsi/celery_executor .

‎build/celeryworker/celeryworker/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
from celeryworker import jobs

‎build/celeryworker/celeryworker/celery.py

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from __future__ import absolute_import
2+
from celery import Celery
3+
import socket
4+
5+
IP = 'rabbit'
6+
PRJ = 'celeryworker'
7+
PORT = '5672'
8+
RABBIT_USER = 'admin'
9+
RABBIT_PASS = 'mypass'
10+
app = Celery(PRJ, broker='amqp://%s:%s@%s:%s' % (RABBIT_USER, RABBIT_PASS,
11+
IP, PORT), backend='rpc://', include=[
12+
'celeryworker.tasks'])

‎build/celeryworker/celeryworker/compat.py

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
from __future__ import absolute_import, print_function
2+
import sys, urllib
3+
is_py2 = sys.version[0] == '2'
4+
if is_py2:
5+
print('Python 2!')
6+
import Queue as queue
7+
input = raw_input
8+
urlencode = urllib.pathname2url
9+
else:
10+
import queue
11+
urlencode = urllib.parse.quote

‎build/celeryworker/celeryworker/jobs.py

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
from __future__ import absolute_import, print_function
2+
import time
3+
from .tasks import *
4+
from .compat import *
5+
6+
registered_images = ['castrona/jhswarm', 'jupyter/scipy-notebook', 'summa']
7+
8+
def get_registered_images():
9+
10+
task = task_get_registered_images.delay()
11+
task = wait_for_task(task)
12+
res = task.result.split('\n')
13+
imgs = [res[i].split()[0] for i in range(1,len(res)-1) if res[i].split()[0] in registered_images]
14+
return imgs
15+
16+
def run(image_name, vol_mount, mount_target, env_vars={}):
17+
18+
invoker_id = os.popen('basename "$(head /proc/1/cgroup)"').read().strip()
19+
print('Name: %s' % image_name)
20+
print('Local Relative Path: %s' % vol_mount)
21+
print('Mount target: %s' % mount_target)
22+
print('Invoking Container Id: %s' % invoker_id)
23+
24+
# make sure the image is registered
25+
if image_name not in registered_images:
26+
print('Cannot run a non-registered container')
27+
return None
28+
29+
task = task_run_container.delay(image_name, vol_mount, mount_target, invoker_id, env_vars)
30+
task = wait_for_task(task)
31+
print('task complete')
32+
print(task.result)
33+
return task
34+
35+
36+
37+
def sanity_check(string, async=True):
38+
if async:
39+
task = task.sanity_check(string)
40+
res = task.result
41+
else:
42+
res = task.sanity_check(string)
43+
return res
44+
45+
def wait_for_task(task):
46+
print('waiting for task to finish', end='')
47+
while not task.ready():
48+
# print(task.info)
49+
print('.', end='')
50+
time.sleep(.25)
51+
print(' done')
52+
return task

‎build/celeryworker/celeryworker/run_tasks.py

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
from celeryworker.tasks import *
2+
import time
3+
import os
4+
from celeryworker.compat import *
5+
6+
if __name__ == '__main__':
7+
8+
volume = os.path.abspath('/share')
9+
print(volume)
10+
11+
# get the id of the current container
12+
image_id = os.popen('basename "$(head /proc/1/cgroup)"').read().strip()
13+
14+
# all paths should be relative to the mounted volume
15+
vars = {'LOCALBASEDIR': 'share/summa_tests',
16+
'OUTDIR': 'output/syntheticTestCases/celia1990/',
17+
'MASTERPATH': 'settings/syntheticTestCases/celia1990/summa_fileManager_celia1990.txt'
18+
}
19+
20+
res = run_container.delay('summa',
21+
volume,
22+
'/tmp/summa',
23+
image_id,
24+
vars)
25+
while not res.ready():
26+
time.sleep(1)
27+
print('...working')
28+
print(res.result)
29+
# print(os.listdir(volume))
30+
print('finished')
31+
32+
33+
34+
#if __name__ == '__main__':
35+
# for _ in xrange(10):
36+
# result = longtime_add.delay(1,2)
37+
# print 'Task finished?',result.ready()
38+
# print 'Task result:',result.result
39+
# time.sleep(1)
40+
# print 'Task finished"',result.ready()
41+
# print 'Task result:',result.result
42+
#

‎build/celeryworker/celeryworker/tasks.py

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
from __future__ import absolute_import, print_function
2+
from celeryworker.celery import app
3+
import time, subprocess, shlex, os
4+
from celeryworker.compat import *
5+
6+
7+
registered_images = ['castrona/jhswarm', 'jupyter/scipy-notebook']
8+
9+
@app.task
10+
def task_sanity_check(string):
11+
time.sleep(1)
12+
print(string)
13+
image_id = os.popen('basename "$(head /proc/1/cgroup)"').read().strip()
14+
print(image_id)
15+
time.sleep(1)
16+
return {'success': 1,'imageid': image_id}
17+
18+
@app.task
19+
def task_get_registered_images():
20+
res = os.popen("docker images").read()
21+
return res
22+
# cmd = 'docker images'
23+
# res = run_command(cmd)
24+
# return res
25+
# res = os.popen("docker images").read().split()
26+
# imgs = [res[i] for i in range(6,len(res),7) if res[i] in registered_images]
27+
# return imgs
28+
29+
@app.task
30+
def task_run_container(image_name, vol_mount, mount_target, invoker_id, env_vars={}):
31+
32+
# print('Name: %s' % image_name)
33+
# print('Local Relative Path: %s' % vol_mount)
34+
# print('Mount target: %s' % mount_target)
35+
# print('Invoking Container Id: %s' % invoker_id)
36+
37+
vols = os.popen("docker inspect -f '{{ .Mounts }}' %s" % invoker_id).read()
38+
volumes = []
39+
for vol in vols.strip()[2:-2].split('} {'):
40+
atts = vol.split(' ')
41+
mnt_path = os.path.join(mount_target, atts[3].strip('/'))
42+
volumes.append('-v ' + (':').join([atts[2], mnt_path]))
43+
44+
for v in volumes:
45+
print(v)
46+
47+
envars = '-e %s=%s' % ('RELPATH',
48+
os.path.join('/tmp', vol_mount.strip('/')))
49+
for k, v in env_vars.items():
50+
envars += ' -e %s=%s' % (k, v)
51+
52+
print(envars)
53+
run_cmd = 'docker run --rm %s %s %s' % ((' ').join(volumes),
54+
envars,
55+
image_name)
56+
print('RUN COMMAND: %s' % run_cmd)
57+
58+
res = os.popen(run_cmd).read()
59+
return res
60+
# res = run_command(run_cmd)
61+
# print('finished. returning output')
62+
# return res
63+
64+
65+
def run_command(command):
66+
process = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
67+
while True:
68+
output = process.stdout.readline()
69+
if output == '' and process.poll() is not None:
70+
break
71+
if output:
72+
print(output.strip())
73+
74+
rc = process.poll()
75+
return rc

‎build/celeryworker/requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
celery==4.0.2

‎build/jupyterhub-rest/Dockerfile

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
FROM ubuntu:xenial
2+
3+
MAINTAINER Tony Castronova <acastronova@cuahsi.org>
4+
5+
6+
ADD install-deps.sh /tmp/install-deps.sh
7+
#ADD utilities.py /tmp/utilities.py
8+
RUN sh /tmp/install-deps.sh
9+
10+
RUN mkdir /app
11+
WORKDIR /app
12+
ADD start.py /app/start.py
13+
14+
ENTRYPOINT python3 start.py
15+
16+

‎build/jupyterhub-rest/build.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
docker build -t cuahsi/jupyterhub-rest .

‎build/jupyterhub-rest/install-deps.sh

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
set -x
3+
set -e
4+
5+
apt-get update
6+
apt-get install --fix-missing -y --no-install-recommends \
7+
git \
8+
vim \
9+
ca-certificates \
10+
python3 \
11+
python3-pip
12+
13+
pip3 install setuptools wheel
14+
15+
cd /srv
16+
git clone https://github.com/hydroshare/hydroshare-jupyterhub.git
17+
#rm hydroshare-jupyterhub/jupyterhub_rest_server/jupyterhub_rest_server/utilities.py
18+
#cp utilities.py hydroshare-jupyterhub/jupyterhub_rest_server/jupyterhub_rest_server/utilities.py
19+
(cd hydroshare-jupyterhub; git pull; pip3 install -e jupyterhub_rest_server)

‎build/jupyterhub-rest/start.py

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env python3
2+
import os
3+
from jupyterhub_rest_server import server
4+
5+
# check that all the required envvars exist
6+
args = ['HYDRPSHARE_REDIRECT_COOKIE', 'JUPYTER_HUB_IP',
7+
'JUPYTER_REST_IP', 'JUPYTER_PORT', 'JUPYTER_REST_PORT',
8+
'JUPYTER_USERSPACE_DIR', 'JUPYTER_USER', 'JUPYTER_NOTEBOOK_DIR']
9+
for a in args:
10+
try:
11+
v = os.environ[a]
12+
except:
13+
print('Missing required EnvVar: %s' % a)
14+
15+
# check that the notebook path exists
16+
dirs = ['JUPYTER_NOTEBOOK_DIR', 'JUPYTER_USERSPACE_DIR']
17+
for d in dirs:
18+
p = os.environ[d]
19+
if not os.path.exists(p):
20+
print('Path does not exist: %s:%s' % (d, p))
21+
22+
# start the server
23+
server.main()

‎build/jupyterhub-rest/utilities.py

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# this file contains utility functions used by the RequestHandlers
2+
3+
import os, stat
4+
from pwd import getpwnam
5+
import grp
6+
import shutil
7+
import logging
8+
9+
log = logging.getLogger()
10+
11+
12+
def set_hydroshare_args(username, resourceid, resourcetype):
13+
14+
userspace_dir = os.environ['JUPYTER_USERSPACE_DIR']
15+
hs_env = os.path.abspath(os.path.join(userspace_dir, '%s/notebooks/.env' % username.lower()))
16+
print('ENV_PATH ',hs_env)
17+
18+
with open(hs_env, 'w') as f:
19+
f.write('HS_USR_NAME=%s\n' % username)
20+
f.write('HS_RES_ID=%s\n' % resourceid)
21+
f.write('HS_RES_TYPE=%s\n' % resourcetype)
22+
f.write('JUPYTER_HUB_IP=%s\n' % os.environ['JUPYTER_HUB_IP'] )
23+
24+
# get the jupyter username
25+
user = getpwnam(os.environ['JUPYTER_USER'])
26+
group = grp.getgrnam('users')
27+
uid = user.pw_uid
28+
gid = group.gr_gid
29+
os.chown(hs_env, uid, gid)
30+
31+
def build_userspace(username):
32+
33+
# make all usernames lowercase
34+
husername = username.lower()
35+
36+
# get the jupyter username
37+
user = getpwnam(os.environ['JUPYTER_USER'])
38+
group = grp.getgrnam('users')
39+
uid = user.pw_uid
40+
gid = group.gr_gid
41+
42+
userspace_dir = os.environ['JUPYTER_USERSPACE_DIR']
43+
ipynb_dir = os.environ['JUPYTER_NOTEBOOK_DIR']
44+
45+
# check to see if user exists
46+
basepath = os.path.abspath(os.path.join(userspace_dir, '%s'%husername))
47+
path = os.path.abspath(os.path.join(basepath, 'notebooks'))
48+
if not os.path.exists(path):
49+
os.makedirs(path)
50+
51+
file_paths = []
52+
print('%s -> copying userpace filse' % username, flush=True)
53+
#ipynb_dir = '../jupyter-rest-endpoint/notebooks'
54+
for root, dirs, files in os.walk(ipynb_dir):
55+
for file in files:
56+
file_paths.append(os.path.join(os.path.abspath(root), file))
57+
relpaths = [os.path.relpath(p, ipynb_dir) for p in file_paths]
58+
for i in range(0, len(file_paths)):
59+
src = file_paths[i]
60+
dst = os.path.join(path, relpaths[i])
61+
dirpath = os.path.dirname(dst)
62+
if not os.path.exists(dirpath):
63+
os.makedirs(dirpath)
64+
shutil.copyfile(src, dst)
65+
66+
# change file ownership so that it can be accessed inside docker container
67+
print('%s -> modifying userspace permissions' % username, flush=True)
68+
os.chown(basepath, uid, gid)
69+
os.chown(os.path.dirname(basepath), uid, gid)
70+
# os.chmod(os.path.dirname(basepath), stat.S_IRWXG | stat.S_ISGID | stat.S_IRWXU)
71+
os.chmod(os.path.dirname(basepath), 0o2770)
72+
73+
for root, dirs, files in os.walk(basepath):
74+
for d in dirs:
75+
os.chown(os.path.join(root, d), uid, gid)
76+
# os.chmod(os.path.join(root, d), stat.S_IRWXG | stat.S_ISGID | stat.S_IRWXU)
77+
os.chmod(os.path.join(root, d), 0o2770)
78+
79+
for f in files:
80+
os.chown(os.path.join(root, f), uid, gid)
81+
# os.chmod(os.path.join(root, f), stat.S_IRWXG | stat.S_ISGID | stat.S_IRWXU)
82+
os.chmod(os.path.join(root, f), 0o2770)

‎build/jupyterhub/Dockerfile

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM jupyterhub/jupyterhub:latest
2+
MAINTAINER Tony Castronova <acastronova@cuahsi.org>
3+
4+
ADD install-deps.sh /tmp/install-deps.sh
5+
RUN sh /tmp/install-deps.sh
6+
7+
ADD cull_idle_servers.py /srv/cull_idle_servers.py

‎build/jupyterhub/build.sh

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
4+
docker build -t cuahsi/jupyterhub .
5+

0 commit comments

Comments
 (0)