diff --git a/.circleci/config.yml b/.circleci/config.yml
index 64db5e0..ff1f283 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -72,6 +72,12 @@ jobs:
                 command: |
                     .circleci/integration-test.py run-test basic-tests test_hub.py test_install.py test_extensions.py
 
+            - run:
+                name: Run admin tests
+                command: |
+                    .circleci/integration-test.py run-test --installer-args "--admin admin:admin" basic-tests test_admin_installer.py
+
+
             - run:
                 name: Run plugin tests
                 command: |
diff --git a/integration-tests/test_admin_installer.py b/integration-tests/test_admin_installer.py
new file mode 100644
index 0000000..3fcc169
--- /dev/null
+++ b/integration-tests/test_admin_installer.py
@@ -0,0 +1,45 @@
+from hubtraf.user import User
+from hubtraf.auth.dummy import login_dummy
+import pytest
+from functools import partial
+import asyncio
+
+
+@pytest.mark.asyncio
+async def test_admin_login():
+    """
+    Test if the admin that was added during install can login with
+    the password provided.
+    """
+    hub_url = 'http://localhost'
+    username = "admin"
+    password = "admin"
+
+    async with User(username, hub_url, partial(login_dummy, password=password)) as u:
+            await u.login()
+            await u.ensure_server()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "username, password",
+    [
+        ("admin", ""),
+        ("admin", "wrong_passw"),
+        ("user", "password"),
+    ],
+)
+async def test_unsuccessful_login(username, password):
+    """
+    Ensure nobody but the admin that was added during install can login
+    """
+    hub_url = 'http://localhost'
+
+    try:
+        async with User(username, hub_url, partial(login_dummy, password="")) as u:
+                await u.login()
+    except Exception:
+        # This is what we except to happen
+        pass
+    else:
+        raise
diff --git a/tests/test_installer.py b/tests/test_installer.py
index 4613cb7..7d594d6 100644
--- a/tests/test_installer.py
+++ b/tests/test_installer.py
@@ -2,6 +2,7 @@
 Unit test  functions in installer.py
 """
 import os
+import pytest
 
 from tljh import installer
 from tljh.yaml import yaml
@@ -21,10 +22,17 @@ def test_ensure_config_yaml(tljh_dir):
     # verify that old config doesn't exist
     assert not os.path.exists(os.path.join(tljh_dir, 'config.yaml'))
 
-def test_ensure_admins(tljh_dir):
+
+@pytest.mark.parametrize(
+	"admins, expected_config",
+	[
+		([['a1'], ['a2'], ['a3']], ['a1', 'a2', 'a3']),
+		([['a1:p1'], ['a2']], ['a1', 'a2']),
+	],
+)
+def test_ensure_admins(tljh_dir, admins, expected_config):
 	# --admin option called multiple times on the installer
 	# creates a list of argument lists.
-	admins = [['a1'], ['a2'], ['a3']]
 	installer.ensure_admins(admins)
 
 	config_path = installer.CONFIG_FILE
@@ -32,4 +40,4 @@ def test_ensure_admins(tljh_dir):
 	    config = yaml.load(f)
 
 	# verify the list was flattened
-	assert config['users']['admin'] == ['a1', 'a2', 'a3']
+	assert config['users']['admin'] == expected_config
diff --git a/tljh/installer.py b/tljh/installer.py
index 48be4d0..322ee45 100644
--- a/tljh/installer.py
+++ b/tljh/installer.py
@@ -129,8 +129,6 @@ def ensure_jupyterhub_service(prefix):
     Ensure JupyterHub Services are set up properly
     """
 
-    os.makedirs(STATE_DIR, mode=0o700, exist_ok=True)
-
     remove_chp()
     systemd.reload_daemon()
 
@@ -278,6 +276,8 @@ def ensure_admins(admin_password_list):
     """
     Setup given list of users as admins.
     """
+    os.makedirs(STATE_DIR, mode=0o700, exist_ok=True)
+
     if not admin_password_list:
         return
     logger.info("Setting up admin users")
@@ -293,8 +293,8 @@ def ensure_admins(admin_password_list):
     db_passw = os.path.join(STATE_DIR, 'passwords.dbm')
 
     admins = []
-    for i in range(len(admin_password_list)):
-        for admin_password_pair in admin_password_list[i]:
+    for admin_password_entry in admin_password_list:
+        for admin_password_pair in admin_password_entry:
             if ":" in admin_password_pair:
                 admin, password = admin_password_pair.split(':')
                 admins.append(admin)