Add better tests for web.py; fix some bugs found with spidering.

- _spider in web.py was actually failing to spider deeper than a certain
  point.
- Fixed multiprocessing pools to not use daemons and to allow recursive
  spawning.
- Added detailed tests for spidering and for finding archive versions.
- left some xfail URL finding exercises for the reader.
- Fix noqa annotations for some @when decorators

var/spack/repos/builtin/packages/qt/package.py

@@ -33,7 +33,7 @@ class Qt(Package):
     homepage = 'http://qt.io'
     url      = 'http://download.qt.io/archive/qt/5.7/5.7.0/single/qt-everywhere-opensource-src-5.7.0.tar.gz'
     list_url = 'http://download.qt.io/archive/qt/'
-    list_depth = 4
+    list_depth = 3
 
     version('5.7.1',  '031fb3fd0c3cc0f1082644492683f18d')
     version('5.7.0',  '9a46cce61fc64c20c3ac0a0e0fa41b42')
@@ -251,7 +251,7 @@ def common_config_args(self):
     # Don't disable all the database drivers, but should
     # really get them into spack at some point.
 
-    @when('@3')
+    @when('@3')  # noqa: F811
     def configure(self):
         # A user reported that this was necessary to link Qt3 on ubuntu.
         # However, if LD_LIBRARY_PATH is not set the qt build fails, check
@@ -268,7 +268,7 @@ def configure(self):
                   '-release',
                   '-fast')
 
-    @when('@4')
+    @when('@4')  # noqa: F811
     def configure(self):
         configure('-fast',
                   '-{0}gtkstyle'.format('' if '+gtk' in self.spec else 'no-'),
@@ -276,7 +276,7 @@ def configure(self):
                   '-arch', str(self.spec.architecture.target),
                   *self.common_config_args)
 
-    @when('@5.0:5.6')
+    @when('@5.0:5.6')  # noqa: F811
     def configure(self):
         webkit_args = [] if '+webkit' in self.spec else ['-skip', 'qtwebkit']
         configure('-no-eglfs',
@@ -284,7 +284,7 @@ def configure(self):
                   '-{0}gtkstyle'.format('' if '+gtk' in self.spec else 'no-'),
                   *(webkit_args + self.common_config_args))
 
-    @when('@5.7:')
+    @when('@5.7:')  # noqa: F811
     def configure(self):
         config_args = self.common_config_args