8th day of python challenges 111-117
This commit is contained in:
abd.shallal
@@ -0,0 +1,337 @@
|
||||
# Copyright (c) 2016-2018 Claudiu Popa <pcmanticore@gmail.com>
|
||||
# Copyright (c) 2016 Derek Gustafson <degustaf@gmail.com>
|
||||
# Copyright (c) 2017 Chris Philip <chrisp533@gmail.com>
|
||||
# Copyright (c) 2017 Hugo <hugovk@users.noreply.github.com>
|
||||
# Copyright (c) 2017 ioanatia <ioanatia@users.noreply.github.com>
|
||||
# Copyright (c) 2017 Calen Pennington <cale@edx.org>
|
||||
# Copyright (c) 2018 Nick Drozd <nicholasdrozd@gmail.com>
|
||||
|
||||
import abc
|
||||
import collections
|
||||
import enum
|
||||
import imp
|
||||
import os
|
||||
import sys
|
||||
import zipimport
|
||||
|
||||
try:
|
||||
import importlib.machinery
|
||||
|
||||
_HAS_MACHINERY = True
|
||||
except ImportError:
|
||||
_HAS_MACHINERY = False
|
||||
|
||||
try:
|
||||
from functools import lru_cache
|
||||
except ImportError:
|
||||
from backports.functools_lru_cache import lru_cache
|
||||
|
||||
from . import util
|
||||
|
||||
ModuleType = enum.Enum(
|
||||
"ModuleType",
|
||||
"C_BUILTIN C_EXTENSION PKG_DIRECTORY "
|
||||
"PY_CODERESOURCE PY_COMPILED PY_FROZEN PY_RESOURCE "
|
||||
"PY_SOURCE PY_ZIPMODULE PY_NAMESPACE",
|
||||
)
|
||||
_ImpTypes = {
|
||||
imp.C_BUILTIN: ModuleType.C_BUILTIN,
|
||||
imp.C_EXTENSION: ModuleType.C_EXTENSION,
|
||||
imp.PKG_DIRECTORY: ModuleType.PKG_DIRECTORY,
|
||||
imp.PY_COMPILED: ModuleType.PY_COMPILED,
|
||||
imp.PY_FROZEN: ModuleType.PY_FROZEN,
|
||||
imp.PY_SOURCE: ModuleType.PY_SOURCE,
|
||||
}
|
||||
if hasattr(imp, "PY_RESOURCE"):
|
||||
_ImpTypes[imp.PY_RESOURCE] = ModuleType.PY_RESOURCE
|
||||
if hasattr(imp, "PY_CODERESOURCE"):
|
||||
_ImpTypes[imp.PY_CODERESOURCE] = ModuleType.PY_CODERESOURCE
|
||||
|
||||
|
||||
def _imp_type_to_module_type(imp_type):
|
||||
return _ImpTypes[imp_type]
|
||||
|
||||
|
||||
_ModuleSpec = collections.namedtuple(
|
||||
"_ModuleSpec", "name type location " "origin submodule_search_locations"
|
||||
)
|
||||
|
||||
|
||||
class ModuleSpec(_ModuleSpec):
|
||||
"""Defines a class similar to PEP 420's ModuleSpec
|
||||
|
||||
A module spec defines a name of a module, its type, location
|
||||
and where submodules can be found, if the module is a package.
|
||||
"""
|
||||
|
||||
def __new__(
|
||||
cls,
|
||||
name,
|
||||
module_type,
|
||||
location=None,
|
||||
origin=None,
|
||||
submodule_search_locations=None,
|
||||
):
|
||||
return _ModuleSpec.__new__(
|
||||
cls,
|
||||
name=name,
|
||||
type=module_type,
|
||||
location=location,
|
||||
origin=origin,
|
||||
submodule_search_locations=submodule_search_locations,
|
||||
)
|
||||
|
||||
|
||||
class Finder:
|
||||
"""A finder is a class which knows how to find a particular module."""
|
||||
|
||||
def __init__(self, path=None):
|
||||
self._path = path or sys.path
|
||||
|
||||
@abc.abstractmethod
|
||||
def find_module(self, modname, module_parts, processed, submodule_path):
|
||||
"""Find the given module
|
||||
|
||||
Each finder is responsible for each protocol of finding, as long as
|
||||
they all return a ModuleSpec.
|
||||
|
||||
:param str modname: The module which needs to be searched.
|
||||
:param list module_parts: It should be a list of strings,
|
||||
where each part contributes to the module's
|
||||
namespace.
|
||||
:param list processed: What parts from the module parts were processed
|
||||
so far.
|
||||
:param list submodule_path: A list of paths where the module
|
||||
can be looked into.
|
||||
:returns: A ModuleSpec, describing how and where the module was found,
|
||||
None, otherwise.
|
||||
"""
|
||||
|
||||
def contribute_to_path(self, spec, processed):
|
||||
"""Get a list of extra paths where this finder can search."""
|
||||
|
||||
|
||||
class ImpFinder(Finder):
|
||||
"""A finder based on the imp module."""
|
||||
|
||||
def find_module(self, modname, module_parts, processed, submodule_path):
|
||||
if submodule_path is not None:
|
||||
submodule_path = list(submodule_path)
|
||||
try:
|
||||
stream, mp_filename, mp_desc = imp.find_module(modname, submodule_path)
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
# Close resources.
|
||||
if stream:
|
||||
stream.close()
|
||||
|
||||
return ModuleSpec(
|
||||
name=modname,
|
||||
location=mp_filename,
|
||||
module_type=_imp_type_to_module_type(mp_desc[2]),
|
||||
)
|
||||
|
||||
def contribute_to_path(self, spec, processed):
|
||||
if spec.location is None:
|
||||
# Builtin.
|
||||
return None
|
||||
|
||||
if _is_setuptools_namespace(spec.location):
|
||||
# extend_path is called, search sys.path for module/packages
|
||||
# of this name see pkgutil.extend_path documentation
|
||||
path = [
|
||||
os.path.join(p, *processed)
|
||||
for p in sys.path
|
||||
if os.path.isdir(os.path.join(p, *processed))
|
||||
]
|
||||
else:
|
||||
path = [spec.location]
|
||||
return path
|
||||
|
||||
|
||||
class ExplicitNamespacePackageFinder(ImpFinder):
|
||||
"""A finder for the explicit namespace packages, generated through pkg_resources."""
|
||||
|
||||
def find_module(self, modname, module_parts, processed, submodule_path):
|
||||
if processed:
|
||||
modname = ".".join(processed + [modname])
|
||||
if util.is_namespace(modname) and modname in sys.modules:
|
||||
submodule_path = sys.modules[modname].__path__
|
||||
return ModuleSpec(
|
||||
name=modname,
|
||||
location="",
|
||||
origin="namespace",
|
||||
module_type=ModuleType.PY_NAMESPACE,
|
||||
submodule_search_locations=submodule_path,
|
||||
)
|
||||
return None
|
||||
|
||||
def contribute_to_path(self, spec, processed):
|
||||
return spec.submodule_search_locations
|
||||
|
||||
|
||||
class ZipFinder(Finder):
|
||||
"""Finder that knows how to find a module inside zip files."""
|
||||
|
||||
def __init__(self, path):
|
||||
super(ZipFinder, self).__init__(path)
|
||||
self._zipimporters = _precache_zipimporters(path)
|
||||
|
||||
def find_module(self, modname, module_parts, processed, submodule_path):
|
||||
try:
|
||||
file_type, filename, path = _search_zip(module_parts, self._zipimporters)
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
return ModuleSpec(
|
||||
name=modname,
|
||||
location=filename,
|
||||
origin="egg",
|
||||
module_type=file_type,
|
||||
submodule_search_locations=path,
|
||||
)
|
||||
|
||||
|
||||
class PathSpecFinder(Finder):
|
||||
"""Finder based on importlib.machinery.PathFinder."""
|
||||
|
||||
def find_module(self, modname, module_parts, processed, submodule_path):
|
||||
spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path)
|
||||
if spec:
|
||||
# origin can be either a string on older Python versions
|
||||
# or None in case it is a namespace package:
|
||||
# https://github.com/python/cpython/pull/5481
|
||||
is_namespace_pkg = spec.origin in ("namespace", None)
|
||||
location = spec.origin if not is_namespace_pkg else None
|
||||
module_type = ModuleType.PY_NAMESPACE if is_namespace_pkg else None
|
||||
spec = ModuleSpec(
|
||||
name=spec.name,
|
||||
location=location,
|
||||
origin=spec.origin,
|
||||
module_type=module_type,
|
||||
submodule_search_locations=list(spec.submodule_search_locations or []),
|
||||
)
|
||||
return spec
|
||||
|
||||
def contribute_to_path(self, spec, processed):
|
||||
if spec.type == ModuleType.PY_NAMESPACE:
|
||||
return spec.submodule_search_locations
|
||||
return None
|
||||
|
||||
|
||||
_SPEC_FINDERS = (ImpFinder, ZipFinder)
|
||||
if _HAS_MACHINERY and sys.version_info[:2] >= (3, 4):
|
||||
_SPEC_FINDERS += (PathSpecFinder,)
|
||||
_SPEC_FINDERS += (ExplicitNamespacePackageFinder,)
|
||||
|
||||
|
||||
def _is_setuptools_namespace(location):
|
||||
try:
|
||||
with open(os.path.join(location, "__init__.py"), "rb") as stream:
|
||||
data = stream.read(4096)
|
||||
except IOError:
|
||||
pass
|
||||
else:
|
||||
extend_path = b"pkgutil" in data and b"extend_path" in data
|
||||
declare_namespace = (
|
||||
b"pkg_resources" in data and b"declare_namespace(__name__)" in data
|
||||
)
|
||||
return extend_path or declare_namespace
|
||||
|
||||
|
||||
@lru_cache()
|
||||
def _cached_set_diff(left, right):
|
||||
result = set(left)
|
||||
result.difference_update(right)
|
||||
return result
|
||||
|
||||
|
||||
def _precache_zipimporters(path=None):
|
||||
pic = sys.path_importer_cache
|
||||
|
||||
# When measured, despite having the same complexity (O(n)),
|
||||
# converting to tuples and then caching the conversion to sets
|
||||
# and the set difference is faster than converting to sets
|
||||
# and then only caching the set difference.
|
||||
|
||||
req_paths = tuple(path or sys.path)
|
||||
cached_paths = tuple(pic)
|
||||
new_paths = _cached_set_diff(req_paths, cached_paths)
|
||||
for entry_path in new_paths:
|
||||
try:
|
||||
pic[entry_path] = zipimport.zipimporter(entry_path)
|
||||
except zipimport.ZipImportError:
|
||||
continue
|
||||
return pic
|
||||
|
||||
|
||||
def _search_zip(modpath, pic):
|
||||
for filepath, importer in list(pic.items()):
|
||||
if importer is not None:
|
||||
found = importer.find_module(modpath[0])
|
||||
if found:
|
||||
if not importer.find_module(os.path.sep.join(modpath)):
|
||||
raise ImportError(
|
||||
"No module named %s in %s/%s"
|
||||
% (".".join(modpath[1:]), filepath, modpath)
|
||||
)
|
||||
# import code; code.interact(local=locals())
|
||||
return (
|
||||
ModuleType.PY_ZIPMODULE,
|
||||
os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath),
|
||||
filepath,
|
||||
)
|
||||
raise ImportError("No module named %s" % ".".join(modpath))
|
||||
|
||||
|
||||
def _find_spec_with_path(search_path, modname, module_parts, processed, submodule_path):
|
||||
finders = [finder(search_path) for finder in _SPEC_FINDERS]
|
||||
for finder in finders:
|
||||
spec = finder.find_module(modname, module_parts, processed, submodule_path)
|
||||
if spec is None:
|
||||
continue
|
||||
return finder, spec
|
||||
|
||||
raise ImportError("No module named %s" % ".".join(module_parts))
|
||||
|
||||
|
||||
def find_spec(modpath, path=None):
|
||||
"""Find a spec for the given module.
|
||||
|
||||
:type modpath: list or tuple
|
||||
:param modpath:
|
||||
split module's name (i.e name of a module or package split
|
||||
on '.'), with leading empty strings for explicit relative import
|
||||
|
||||
:type path: list or None
|
||||
:param path:
|
||||
optional list of path where the module or package should be
|
||||
searched (use sys.path if nothing or None is given)
|
||||
|
||||
:rtype: ModuleSpec
|
||||
:return: A module spec, which describes how the module was
|
||||
found and where.
|
||||
"""
|
||||
_path = path or sys.path
|
||||
|
||||
# Need a copy for not mutating the argument.
|
||||
modpath = modpath[:]
|
||||
|
||||
submodule_path = None
|
||||
module_parts = modpath[:]
|
||||
processed = []
|
||||
|
||||
while modpath:
|
||||
modname = modpath.pop(0)
|
||||
finder, spec = _find_spec_with_path(
|
||||
_path, modname, module_parts, processed, submodule_path or path
|
||||
)
|
||||
processed.append(modname)
|
||||
if modpath:
|
||||
submodule_path = finder.contribute_to_path(spec, processed)
|
||||
|
||||
if spec.type == ModuleType.PKG_DIRECTORY:
|
||||
spec = spec._replace(submodule_search_locations=submodule_path)
|
||||
|
||||
return spec
|
||||
@@ -0,0 +1,10 @@
|
||||
# Copyright (c) 2016, 2018 Claudiu Popa <pcmanticore@gmail.com>
|
||||
|
||||
try:
|
||||
import pkg_resources
|
||||
except ImportError:
|
||||
pkg_resources = None
|
||||
|
||||
|
||||
def is_namespace(modname):
|
||||
return pkg_resources is not None and modname in pkg_resources._namespace_packages
|
||||
Reference in New Issue
Block a user