3
A|e @ s d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl
Z
ddl Z ddl Z ddl
ZddlZddlZddlZddlmZ ddlZddlmZmZmZmZmZm Z m Z m Z m Z m Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z, dd l-m.Z. ej/d
Z0ej/d ej1Z2ej/d Z3ej/d
ej1j4Z5dj6 Z7ddddgZ8dZ9dZ:e:j;dj;ej< edZ=dd Z>dd Z?dd Z@dAd dZAdBd d ZBdCd d ZCdedfd!dZDd"d# ZEej/d$ej1ZFeEd%d& ZGG d'd( d(ZHG d)d* d*eHZIG d+d de ZJej/d,jKZLd-d. ZMd/d0 ZNdDd1d2ZOd3d4 ZPG d5d6 d6ZQG d7d8 d8e jRZSejTjUfd9d:ZVd;d< ZWeOe9eVZVd=d> ZXd?d@ ZYdS )Ez#PyPI and direct package downloading N)wraps)
CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST
parse_version)log)DistutilsError) translate)Wheel)unique_everseenz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)\n\s+\(md5\)z([-+.a-z0-9]{2,}):z .tar.gz .tar.bz2 .tar .zip .tgz PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_name z s z(interpret_distro_name..r5 Nr3 )
py_versionrF rW )r< anyrangerT r join)rN rU rK r` rF rW r> r^ r r r! r s
$c s t fdd}|S )zs
Wrap a function returning an iterable such that the resulting iterable
only ever yields unique items.
c s t | |S )N)r )argskwargs)funcr r! wrapper s z unique_values..wrapper)r )rf rg r )rf r!
unique_values s rh z(<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>c c s xvt j|D ]h}|j \}}tttj|j jd}d|ksFd|kr x,t j|D ] }t
j j | t
|jdV qRW q W xHdD ]@}|j|}|d kr~t j||}|r~t
j j | t
|jdV q~W dS )
zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,homepager4 r3
| Home Page | Download URLN)rk rl r7 )RELfinditergroupssetmapstrstripr+ r< HREFr9 r urljoin
htmldecoderJ findsearch)r= pagerI tagrelZrelsposr r r! find_external_links s "
r} c @ s( e Zd ZdZdd Zdd Zdd ZdS ) ContentCheckerzP
A null content checker that defines the interface for checking content
c C s dS )z3
Feed a block of data to the hash.
Nr )selfblockr r r! feed s zContentChecker.feedc C s dS )zC
Check the hash. Return False if validation fails.
Tr )r r r r! is_valid s zContentChecker.is_validc C s dS )zu
Call reporter with information about the checker (hash name)
substituted into the template.
Nr )r reportertemplater r r! report s zContentChecker.reportN)__name__
__module__ __qualname____doc__r r r r r r r! r~ s r~ c @ sB e Zd ZejdZdd Zedd Zdd Z dd Z
d
d Z d S )
HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)c C s || _ tj|| _|| _d S )N) hash_namehashlibnewhashexpected)r r r r r r! __init__ s zHashChecker.__init__c C s> t jj|d }|st S | jj|}|s0t S | f |j S )z5Construct a (possibly null) ContentChecker from a URLr3 r7 )r9 r r: r~ patternrx groupdict)clsr= rD rI r r r! from_url s zHashChecker.from_urlc C s | j j| d S )N)r update)r r r r r! r s zHashChecker.feedc C s | j j | jkS )N)r hexdigestr )r r r r! r s zHashChecker.is_validc C s || j }||S )N)r )r r r msgr r r! r s
zHashChecker.reportN) r r r r\ compiler r classmethodr r r r r r r r! r s r c sP e Zd ZdZdOddZ fdd ZdPd d ZdQd
dZdRddZdd Z dd Z
dd Z dd Z dd Z
dSdd Zd d ZdT fd d Zd!d" Zd#d$ Zd%d& Zd'd( Zd)d* ZdUd+d,ZdVd-d.Zd/d0 Zd1Zd2d3 Zd4d5 ZdWd6d7Z d8d9 Z d:d; Z dd? Z d@dA Z!e"dXdBdCZ#dDdE Z$dFdG Z%dHdI Z&dJdK Z'dLdM Z( Z)S )Yr z;A distribution index that scans web pages for download URLshttps://pypi.org/simple/*NTc O sl t j| f|| |dd |jd | _i | _i | _i | _tjdj t
t |j | _
g | _tjj| _d S )Nr2 |)r r r, index_url scanned_urls fetched_urls
package_pagesr\ r rc rq r rI allowsto_scanr9 requesturlopenopener)r r hostsZ ca_bundleZ
verify_sslrd kwr r r! r s zPackageIndex.__init__c s0 yt |j W n tk
r" d S X t j|S )N)r rP Exceptionsuperadd)r rL ) __class__r r! r ) s
zPackageIndex.addFc C s || j kr| rdS d| j |< t|s4| j| dS tt|}|r^| j|sRdS | jd| |sr| sr|| jkrtt| j | dS | j|sd| j|< dS | j
d| d| j|< d}| j ||| }|dkrdS t |t
jjo|jdk r| j
d|j d| j|j< d|jjd d
j kr.|j dS |j}|j }t |ts|t |t
jjr^d }n|jjd pnd }|j|d
}|j x6tj |D ](} t
j j |t | j d}
| j!|
qW |j"| j#rt$|dddkr| j%||}dS )z.)filterrY rA r r itertoolsstarmap
scan_egg_link)r search_pathdirsZ egg_linksr r r! scan_egg_links s zPackageIndex.scan_egg_linksc
C s t tjj|| }ttd ttj|}W d Q R X t |dkrDd S |\}}x>t
tjj||D ](}tjj|f| |_ t |_
| j| q`W d S )Nr5 )openrY rA rc r r rq rr rs rT r rN r rF r )r rA r Z raw_lineslinesegg_pathZ
setup_pathrL r r r! r s zPackageIndex.scan_egg_linkc C s d}|j | js|S tttjj|t| jd jd}t|dksRd|d krV|S t |d }t
|d }d| j j |j
i |< t|t|fS )Nr2 r5 r6 r3 r T)NN)r- r r rq r9 r r; rT r< r
r r
setdefaultr+ r )r r ZNO_MATCH_SENTINELr> pkgverr r r! _scan s " zPackageIndex._scanc
C s xJt j|D ]<}y"| jtjj|t|jd W q tk
rF Y q X q W | j|\}}|sbdS xXt ||D ]J}t
|\}}|j dr| r|r|d||f 7 }n
| j | | j
| qnW tjdd |S )z#Process the contents of a PyPI pager3 r z.pyz
#egg=%s-%sc S s d| j ddd S )Nz%sr3 r5 )rJ )mr r r! s z,PackageIndex.process_index..)rt rn r r9 r ru rv rJ r r} rE r, need_version_infoscan_urlPYPI_MD5sub) r r= ry rI r r new_urlr/ fragr r r! r s" "
zPackageIndex.process_indexc C s | j d| d S )NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_all)r r= r r r! r s z PackageIndex.need_version_infoc G s: | j | jkr*|r | j|f| | jd | j| j d S )Nz6Scanning index of all packages (this may take a while))r r r r r )r r rd r r r! r s zPackageIndex.scan_allc C s~ | j | j|j d | jj|js:| j | j|j d | jj|jsR| j| x&t| jj|jf D ]}| j | qhW d S )Nr2 ) r r unsafe_namer r keyrO not_found_in_indexr )r requirementr= r r r!
find_packages s
zPackageIndex.find_packagesc sR | j | j| x,| |j D ] }||kr.|S | jd|| q W tt| j||S )Nz%s does not match %s)prescanr r r r r obtain)r r installerrL )r r r! r s
zPackageIndex.obtainc C sL |j | jd| |j sH|j tj| td|jjtj j
|f dS )z-
checker is a ContentChecker
z Validating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) r r r r rY unlinkr r r. rA rU )r checkerrZ tfpr r r!
check_hash s
zPackageIndex.check_hashc C sT xN|D ]F}| j dks4t| s4|jds4tt|r@| j| q| j j| qW dS )z;Add `urls` to the list that will be prescanned for searchesNzfile:)r r r- r r r append)r Zurlsr= r r r! add_find_links s
zPackageIndex.add_find_linksc C s" | j rtt| j| j d| _ dS )z7Scan urls scheduled for prescanning (e.g. --find-links)N)r r rq r )r r r r! r s zPackageIndex.prescanc C s< | |j r| jd }}n | jd }}|||j | j d S )Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))r r r r r )r r methr r r r! r s
z PackageIndex.not_found_in_indexc C s~ t |tsjt|}|rR| j|jd||}t|\}}|jdrN| j|||}|S tj j
|rb|S t |}t | j
||ddS )a Locate and/or download `spec` to `tmpdir`, returning a local path
`spec` may be a ``Requirement`` object, or a string containing a URL,
an existing local filename, or a project/version requirement spec
(i.e. the string form of a ``Requirement`` object). If it is the URL
of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
automatically created alongside the downloaded file.
If `spec` is a ``Requirement`` object or a string containing a
project/version requirement spec, this method returns the location of
a matching distribution (possibly after downloading it to `tmpdir`).
If `spec` is a locally existing file or directory name, it is simply
returned unchanged. If `spec` is a URL, it is downloaded to a subpath
of `tmpdir`, and the local filename is returned. Various errors may be
raised if a problem occurs during downloading.
r3 z.pyrN N)r r
r
_download_urlrJ rE r, gen_setuprY rA r r" r fetch_distribution)r r tmpdirr? foundr/ rD r r r! r4 # s
zPackageIndex.downloadc s j d| i d}d
fdd }|rHj j| ||}| r`|dk r`|||}|dkrjdk rzj ||}|dkr| rj| ||}|dkrjdrdpd| nj d| |j|jd S dS ) a| Obtain a distribution suitable for fulfilling `requirement`
`requirement` must be a ``pkg_resources.Requirement`` instance.
If necessary, or if the `force_scan` flag is set, the requirement is
searched for in the (online) package index as well as the locally
installed packages. If a distribution matching `requirement` is found,
the returned distribution's ``location`` is the value you would have
gotten from calling the ``download()`` method with the matching
distribution's URL or filename. If no matching distribution is found,
``None`` is returned.
If the `source` flag is set, only source distributions and source
checkout links will be considered. Unless the `develop_ok` flag is
set, development and system eggs (i.e., those using the ``.egg-info``
format) will be ignored.
zSearching for %sNc s |d kr }x|| j D ]t}|jtkrJ rJ|krjd| d|< q|| ko`|jtkp` }|rj|j}||_tj j
|jr|S qW d S )Nz&Skipping development or system egg: %sr3 ) r rF r r r r4 rN download_locationrY rA r )reqenvrL testloc)
develop_okr skippedsourcer r r! rw ] s z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of r zBest match: %s)rN )N)r r r r r cloner ) r r r
force_scanr r local_indexrL rw r )r r r r r r! r E s0
z PackageIndex.fetch_distributionc C s" | j ||||}|dk r |jS dS )a3 Obtain a file suitable for fulfilling `requirement`
DEPRECATED; use the ``fetch_distribution()`` method now instead. For
backward compatibility, this routine is identical but returns the
``location`` of the downloaded distribution instead of a distribution
object.
N)r rN )r r r r
r rL r r r! fetch s zPackageIndex.fetchc
C s t j|}|r*dd t||jdd D p,g }t|dkrtjj|}tjj||krtjj ||}ddl
m } |||st j
|| |}ttjj |dd2} | jd|d j|d jtjj|d f W d Q R X |S |rtd ||f ntd
d S ) Nc S s g | ]}|j r|qS r )rP )r] dr r r!
s z*PackageIndex.gen_setup..r3 r )samefilezsetup.pywzIfrom setuptools import setup
setup(name=%r, version=%r, py_modules=[%r])
zCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rH rI r rJ rT rY rA rU dirnamerc setuptools.command.easy_installr shutilcopy2r writerO rP splitextr )
r rZ rD r rI r rU dstr r r r r! r s2
zPackageIndex.gen_setupi c C s | j d| d }ztj|}| j|}t|tjjrJtd||j |j
f |j }d}| j }d}d|kr|j d} t
tt| }| j||||| t|dZ}
xD|j|} | r|j| |
j| |d7 }| j||||| qP qW | j|||
W d Q R X |S |r|j X d S ) NzDownloading %szCan't download %s: %s %sr r3 zcontent-lengthzContent-Lengthwbr7 )r r r r r r9 r r r r r dl_blocksizeget_allmaxrq int
reporthookr r r r r r ) r r= rZ fpr r blocknumbssizesizesr r r r r! _download_to s:
zPackageIndex._download_toc C s d S )Nr )r r= rZ r Zblksizer! r r r! r s zPackageIndex.reporthookc C s |j drt|S y t|| jS ttjjfk
r } z@djdd |j D }|r`| j
|| nt d||f |W Y d d }~X n t j
jk
r } z|S d }~X n t j
jk
r } z.|r| j
||j nt d||jf |W Y d d }~X n tjjk
rB } z0|r | j
||j nt d||jf |W Y d d }~X nT tjjtj
fk
r } z,|rr| j
|| nt d||f |W Y d d }~X nX d S )Nzfile: c S s g | ] }t |qS r )rr )r] argr r r! r s z)PackageIndex.open_url..z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r-
local_openopen_with_authr r httpclient
InvalidURLrc rd r r r9 r r URLErrorreasonZ
BadStatusLinelineZ
HTTPExceptionsocket)r r= warningvr r r r! r s8
$ zPackageIndex.open_urlc C s t |\}}|r4x&d|kr0|jddjdd}qW nd}|jdrN|d d }tjj||}|dksn|jd rz| j||S |d
ks|jd r| j||S |jd r| j ||S |d
krt
j j t
j
j|d S | j|d | j||S d S )Nz...\_Z__downloaded__z.egg.zipr' svnzsvn+gitzgit+zhg+r r5 Tr* )rE replacer, rY rA rc r-
_download_svn
_download_git _download_hgr9 r url2pathnamer r: r _attempt_download)r r? r= r r. rD rZ r r r! r s$
zPackageIndex._download_urlc C s | j |d d S )NT)r )r r= r r r! r 1 s zPackageIndex.scan_urlc C s6 | j ||}d|jddj kr.| j|||S |S d S )Nr z content-typer )r# r r+ _download_html)r r= rZ r r r r! r; 4 s z PackageIndex._attempt_downloadc C sl t |}x@|D ]8}|j rtjd|rD|j tj| | j||S P qW |j tj| td| d S )Nz ([^- ]+ - )?Revision \d+:z Unexpected HTML page found at ) r rs r\ rx r rY r r7 r )r r= r rZ r r- r r r! r< ; s
zPackageIndex._download_htmlc C s
t jdt |jddd }d}|j jdrd|krtjj|\}}}}}} | r|jdrd |d
d kr|d
d jd d\}}t |\}
} |
rd |
kr|
jd d\} }
d | |
f }nd
|
}| }|||||| f}tjj
|}| j d|| t j
d|||f |S )Nz"SVN download support is deprecatedr6 r3 r r zsvn:@z//r2 r5 :z --username=%s --password=%sz --username=z'Doing subversion checkout from %s to %szsvn checkout%s -q %s %s)warningsr UserWarningr< r+ r- r9 r r:
_splituser
urlunparser rY system)r r= rZ Zcredsr? netlocrA r^ qr authhostuserZpwr> r r r! r7 J s& zPackageIndex._download_svnc C sp t jj| \}}}}}|jddd }|jddd }d }d|krR|jdd\}}t jj||||df} | |fS )N+r3 r6 r r= r r7 )r9 r urlsplitr< rsplit
urlunsplit)r=
pop_prefixr? rD rA rC r revr r r! _vcs_split_rev_from_url` s z$PackageIndex._vcs_split_rev_from_urlc C sl |j ddd }| j|dd\}}| jd|| tjd||f |d k rh| jd| tjd ||f |S )
Nr6 r3 r T)rM z Doing git clone from %s to %szgit clone --quiet %s %szChecking out %sz git -C %s checkout --quiet %s)r< rO r rY rC )r r= rZ rN r r r! r8 r s
zPackageIndex._download_gitc C sl |j ddd }| j|dd\}}| jd|| tjd||f |d k rh| jd| tjd ||f |S )
Nr6 r3 r T)rM z Doing hg clone from %s to %szhg clone --quiet %s %szUpdating to %szhg --cwd %s up -C -r %s -q)r< rO r rY rC )r r= rZ rN r r r! r9 s
zPackageIndex._download_hgc G s t j|f| d S )N)r r )r r rd r r r! r s zPackageIndex.debugc G s t j|f| d S )N)r r )r r rd r r r! r s zPackageIndex.infoc G s t j|f| d S )N)r r )r r rd r r r! r s zPackageIndex.warnr )r rP NT)F)F)F)N)N)FFFN)FF)N)F)*r r r r r r r r r r r r r r r r r r r r r r4 r r r r r# r r r r r; r< r7 staticmethodrO r8 r9 r r r
__classcell__r r )r r! r sP
5
#
J
)$
# z!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?c C s | j d}tj|S )Nr )rJ r Zunescape)rI whatr r r!
decode_entity s
rT c C s
t t| S )a
Decode HTML entities in the given text.
>>> htmldecode(
... 'https://../package_name-0.1.2.tar.gz'
... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz')
'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz'
)
entity_subrT )textr r r! rv s rv c s fdd}|S )Nc s fdd}|S )Nc s. t j }t j z
| |S t j| X d S )N)r. getdefaulttimeoutsetdefaulttimeout)rd re Z old_timeout)rf timeoutr r! _socket_timeout s
z@socket_timeout.<locals>._socket_timeout.<locals>._socket_timeoutr )rf rZ )rY )rf r! rZ s z'socket_timeout.<locals>._socket_timeoutr )rY rZ r )rY r! socket_timeout s r[ c C s2 t jj| }|j }tj|}|j }|jddS )a9
Encode auth from a URL suitable for an HTTP header.
>>> str(_encode_auth('username%3Apassword'))
'dXNlcm5hbWU6cGFzc3dvcmQ='
Long auth strings should not cause a newline to be inserted.
>>> long_auth = 'username:' + 'password'*10
>>> chr(10) in str(_encode_auth(long_auth))
False
r )r9 r r; encodebase64 b64encoder r6 )rF Zauth_sZ
auth_bytesZ
encoded_bytesencodedr r r! _encode_auth s
ra c @ s( e Zd ZdZdd Zdd Zdd ZdS )
Credentialz:
A username/password pair. Use like a namedtuple.
c C s || _ || _d S )N)usernamepassword)r rc rd r r r! r s zCredential.__init__c c s | j V | jV d S )N)rc rd )r r r r! __iter__ s zCredential.__iter__c C s dt | S )Nz%(username)s:%(password)s)vars)r r r r! __str__ s zCredential.__str__N)r r r r r re rg r r r r! rb s rb c @ s0 e Zd Zdd Zedd Zdd Zdd Zd S )
PyPIConfigc C sP t jdddgd}tjj| | tjjtjjdd}tjj |rL| j
| dS )z%
Load from ~/.pypirc
rc rd
repositoryr ~z.pypircN) dictfromkeys configparserRawConfigParserr rY rA rc
expanduserr r )r defaultsrcr r r! r s
zPyPIConfig.__init__c s& fdd j D }tt j|S )Nc s g | ]} j |d j r|qS )ri )r rs )r] section)r r r! r s z2PyPIConfig.creds_by_repository.<locals>.<listcomp>)sectionsrk rq _get_repo_cred)r Zsections_with_repositoriesr )r r! creds_by_repository s z PyPIConfig.creds_by_repositoryc C s6 | j |dj }|t| j |dj | j |dj fS )Nri rc rd )r rs rb )r rr Zrepor r r! rt s zPyPIConfig._get_repo_credc C s* x$| j j D ]\}}|j|r |S q W dS )z
If the URL indicated appears to be a repository defined in this
config, return the credential for that repository.
N)ru itemsr- )r r= ri credr r r! find_credential s
zPyPIConfig.find_credentialN)r r r r propertyru rt rx r r r r! rh s rh c C s< t jj| }|\}}}}}}|jdr2tjjd|d
krHt|\} }
nd} | st j | } | rt
| } | j | f} t j
d | | rdt| } ||
||||f}
t jj|
}t jj|}|jd| n t jj| }|jd t ||}| r8t jj|j\}}}}}}||kr8||
kr8||||||f}
t jj|
|_|S ) z4Open a urllib2 request, handling HTTP authenticationr> znonnumeric port: ''r( httpsN*Authenticating as %s for %s (from .pypirc)zBasic Z
Authorizationz
User-Agent)r( rz )r{ )r9 r r: r, r( r) r* rA rh rx rr rc r r ra rB r ZRequest
add_header
user_agentr= )r= r parsedr? rD rA paramsrC r rF addressrw r r> r r r s2Zh2Zpath2Zparam2Zquery2Zfrag2r r r! r'
s8
r' c C s | j d\}}} |r|nd| fS )zNsplituser('user[:passwd]@host[:port]')
--> 'user[:passwd]', 'host[:port]'.r= N)
rpartition)rG rH delimr r r! rA < s rA c C s | S )Nr )r= r r r!
fix_sf_urlG s r c C s t jj| \}}}}}}t jj|}tjj|r<t jj| S |j drtjj
|rg }xtj |D ]b} tjj || }
| dkrt
|
d} | j } W dQ R X P ntjj
|
r| d7 } |jdj| d qbW d}
|
j| dj |d } d\}}n
d\}}} ddi}tj| }t jj| ||||S )z7Read a local path, with special support for directoriesr2 z
index.htmlrNz<a href="{name}">{name}</a>)r. zB<html><head><title>{url}{files} |