Merge pull request #163 from minrk/config-dir

put config in `$tljh/config` directory

tljh/config.py

@@ -27,7 +27,8 @@ INSTALL_PREFIX = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh')
 HUB_ENV_PREFIX = os.path.join(INSTALL_PREFIX, 'hub')
 USER_ENV_PREFIX = os.path.join(INSTALL_PREFIX, 'user')
 STATE_DIR = os.path.join(INSTALL_PREFIX, 'state')
-CONFIG_FILE = os.path.join(INSTALL_PREFIX, 'config.yaml')
+CONFIG_DIR = os.path.join(INSTALL_PREFIX, 'config')
+CONFIG_FILE = os.path.join(CONFIG_DIR, 'config.yaml')
 
 
 def set_item_in_config(config, property_path, value):
@@ -221,6 +222,9 @@ def main(argv=None):
     if argv is None:
         argv = sys.argv[1:]
 
+    from .log import init_logging
+    init_logging()
+
     argparser = argparse.ArgumentParser()
     argparser.add_argument(
         '--config-path',

tljh/installer.py

@@ -1,38 +1,44 @@
+"""Installation logic for TLJH"""
+
 import argparse
+import itertools
+import logging
 import os
 import secrets
 import subprocess
-import itertools
 import sys
 import time
-import logging
 from urllib.error import HTTPError
 from urllib.request import urlopen, URLError
-import pluggy
 
+import pluggy
 from ruamel.yaml import YAML
 
-from tljh import conda, systemd, traefik, user, apt, hooks
-from tljh.config import INSTALL_PREFIX, HUB_ENV_PREFIX, USER_ENV_PREFIX, STATE_DIR, CONFIG_FILE
+from tljh import (
+    apt,
+    conda,
+    hooks,
+    migrator,
+    systemd,
+    traefik,
+    user,
+)
+
+from tljh.config import (
+    CONFIG_DIR,
+    CONFIG_FILE,
+    HUB_ENV_PREFIX,
+    INSTALL_PREFIX,
+    STATE_DIR,
+    USER_ENV_PREFIX,
+)
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 
 rt_yaml = YAML()
 
-# Set up logging to print to a file and to stderr
 logger = logging.getLogger(__name__)
 
-os.makedirs(INSTALL_PREFIX, exist_ok=True)
-file_logger = logging.FileHandler(os.path.join(INSTALL_PREFIX, 'installer.log'))
-file_logger.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
-logger.addHandler(file_logger)
-
-stderr_logger = logging.StreamHandler()
-stderr_logger.setFormatter(logging.Formatter('%(message)s'))
-logger.addHandler(stderr_logger)
-logger.setLevel(logging.INFO)
-
-
 def ensure_node():
     """
     Ensure nodejs from nodesource is installed
@@ -254,7 +260,7 @@ def ensure_admins(admins):
     if not admins:
         return
     logger.info("Setting up admin users")
-    config_path = os.path.join(INSTALL_PREFIX, 'config.yaml')
+    config_path = CONFIG_FILE
     if os.path.exists(config_path):
         with open(config_path, 'r') as f:
             config = rt_yaml.load(f)
@@ -324,7 +330,7 @@ def ensure_symlinks(prefix):
     os.symlink(tljh_config_src, tljh_config_dest)
 
 
-def setup_plugins(plugins):
+def setup_plugins(plugins=None):
     """
     Install plugins & setup a pluginmanager
     """
@@ -339,6 +345,7 @@ def setup_plugins(plugins):
 
     return pm
 
+
 def run_plugin_actions(plugin_manager, plugins):
     """
     Run installer hooks defined in plugins
@@ -373,6 +380,12 @@ def ensure_config_yaml(plugin_manager):
     """
     Ensure we have a config.yaml present
     """
+    # ensure config dir exists and is private
+    for path in [CONFIG_DIR, os.path.join(CONFIG_DIR, 'jupyterhub_config.d')]:
+        os.makedirs(path, mode=0o700, exist_ok=True)
+
+    migrator.migrate_config_files()
+
     if os.path.exists(CONFIG_FILE):
         with open(CONFIG_FILE, 'r') as f:
             config = rt_yaml.load(f)
@@ -387,6 +400,9 @@ def ensure_config_yaml(plugin_manager):
 
 
 def main():
+    from .log import init_logging
+    init_logging()
+
     argparser = argparse.ArgumentParser()
     argparser.add_argument(
         '--admin',
@@ -407,8 +423,8 @@ def main():
 
     pm = setup_plugins(args.plugin)
 
+    ensure_config_yaml(pm)
     ensure_admins(args.admin)
-
     ensure_usergroups()
     ensure_user_environment(args.user_requirements_txt_url)
 
@@ -416,7 +432,6 @@ def main():
     ensure_node()
     ensure_jupyterhub_package(HUB_ENV_PREFIX)
     ensure_chp_package(HUB_ENV_PREFIX)
-    ensure_config_yaml(pm)
     ensure_jupyterlab_extensions()
     ensure_jupyterhub_service(HUB_ENV_PREFIX)
     ensure_jupyterhub_running()

tljh/jupyterhub_config.py

@@ -8,7 +8,7 @@ from glob import glob
 
 from systemdspawner import SystemdSpawner
 from tljh import user, configurer
-from tljh.config import INSTALL_PREFIX, USER_ENV_PREFIX
+from tljh.config import INSTALL_PREFIX, USER_ENV_PREFIX, CONFIG_DIR
 
 
 class CustomSpawner(SystemdSpawner):
@@ -43,7 +43,7 @@ c.SystemdSpawner.default_shell = '/bin/bash'
 # Drop the '-singleuser' suffix present in the default template
 c.SystemdSpawner.unit_name_template = 'jupyter-{USERNAME}'
 
-config_overrides_path = os.path.join(INSTALL_PREFIX, 'config.yaml')
+config_overrides_path = os.path.join(CONFIG_DIR, 'config.yaml')
 if os.path.exists(config_overrides_path):
     with open(config_overrides_path) as f:
         config_overrides = yaml.safe_load(f)
@@ -53,6 +53,6 @@ configurer.apply_config(config_overrides, c)
 
 # Load arbitrary .py config files if they exist.
 # This is our escape hatch
-extra_configs = sorted(glob(os.path.join(INSTALL_PREFIX, 'jupyterhub_config.d', '*.py')))
+extra_configs = sorted(glob(os.path.join(CONFIG_DIR, 'jupyterhub_config.d', '*.py')))
 for ec in extra_configs:
     load_subconfig(ec)

tljh/log.py

@@ -0,0 +1,19 @@
+"""Setup tljh logging"""
+import os
+import logging
+
+from .config import INSTALL_PREFIX
+
+
+def init_logging():
+    """Setup default tljh logger"""
+    logger = logging.getLogger("tljh")
+    os.makedirs(INSTALL_PREFIX, exist_ok=True)
+    file_logger = logging.FileHandler(os.path.join(INSTALL_PREFIX, "installer.log"))
+    file_logger.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
+    logger.addHandler(file_logger)
+
+    stderr_logger = logging.StreamHandler()
+    stderr_logger.setFormatter(logging.Formatter("%(message)s"))
+    logger.addHandler(stderr_logger)
+    logger.setLevel(logging.INFO)

tljh/migrator.py

@@ -0,0 +1,69 @@
+"""Migration utilities for upgrading tljh"""
+
+import os
+from datetime import date
+import logging
+import shutil
+
+from tljh.config import (
+    CONFIG_DIR,
+    CONFIG_FILE,
+    INSTALL_PREFIX,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_file(old_path, new_path):
+    """Migrate one file from an old location to a new one
+
+    avoids collisions if the new file exists
+    """
+    if not os.path.exists(old_path):
+        return
+    if os.path.exists(new_path):
+        # new config file already created! still move the config,
+        # but avoid collision
+        timestamp = date.today().isoformat()
+        dest = dest_base = f"{new_path}.old.{timestamp}"
+        i = 0
+        while os.path.exists(dest):
+            # avoid collisions
+            dest = dest_base + f".{i}"
+            i += 1
+        logger.warning(f"Found file in both old ({old_path}) and new ({new_path}).")
+        logger.warning(
+            f"Moving {old_path} to {dest} to avoid clobbering.  Its contents will be ignored."
+        )
+    else:
+        dest = new_path
+    shutil.move(old_path, dest)
+
+
+def migrate_directory(old_dir, new_dir):
+    """Migrate a directory to a new location"""
+    if not os.path.exists(old_dir):
+        return
+    if os.path.exists(new_dir):
+        # both dirs exist
+        for f in os.listdir(old_dir):
+            src = os.path.join(old_dir, f)
+            dest = os.path.join(new_dir, f)
+            if os.path.isdir(src):
+                migrate_directory(src, dest)
+            else:
+                migrate_file(src, dest)
+    else:
+        logger.warning(f"Moving directory to new location {old_dir} -> {new_dir}")
+        shutil.move(old_dir, new_dir)
+
+
+def migrate_config_files():
+    """Migrate config files to their new locations"""
+    # handle old TLJH_DIR/config.yaml location
+    migrate_file(os.path.join(INSTALL_PREFIX, "config.yaml"), CONFIG_FILE)
+    migrate_directory(
+        os.path.join(INSTALL_PREFIX, "jupyterhub_config.d"),
+        os.path.join(CONFIG_DIR, "jupyterhub_config.d"),
+    )