3 jf@sddlmZmZmZddlmZddlmZddlm Z m Z e e_ e e_ dgZ GdddeZGdddeZGd d d eZd S) )absolute_importdivisionunicode_literals)str)urllib)parserequestRobotFileParserc@sZeZdZdZdddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ dS)r zs This class provides a set of methods to read, parse and answer questions about a single robots.txt file. cCs,g|_d|_d|_d|_|j|d|_dS)NFr)entries default_entry disallow_all allow_allset_url last_checked)selfurlrE/tmp/pip-build-s1i5078x/future/future/backports/urllib/robotparser.py__init__s  zRobotFileParser.__init__cCs|jS)zReturns the time the robots.txt file was last fetched. This is useful for long-running web spiders that need to check for new robots.txt files periodically. )r)rrrrmtime&szRobotFileParser.mtimecCsddl}|j|_dS)zYSets the time the robots.txt file was last fetched to the current time. rN)timer)rrrrrmodified/szRobotFileParser.modifiedcCs&||_tjj|dd\|_|_dS)z,Sets the URL referring to a robots.txt file.N)rrrurlparsehostpath)rrrrrr7szRobotFileParser.set_urlcCs|ytjj|j}WnHtjjk rZ}z(|jdkr:d|_n|jdkrJd|_WYdd}~XnX|j }|j |j dj dS)z4Reads the robots.txt URL and feeds it to the parser.TiNzutf-8)rr) rrurlopenrerror HTTPErrorcoder rreadrdecode splitlines)rferrrawrrrr$<s  zRobotFileParser.readcCs,d|jkr|jdkr(||_n |jj|dS)N*) useragentsr r append)rentryrrr _add_entryIs  zRobotFileParser._add_entrycCstd}t}xN|D]D}|sL|dkr0t}d}n|dkrL|j|t}d}|jd}|dkrj|d|}|j}|sxq|jdd}t|dkr|djj|d<tjj |dj|d<|ddkr|dkr|j|t}|j j |dd}q|ddkr*|dkrX|j j t |dd d}q|dd kr|dkr|j j t |dd d}qW|dkrp|j|dS) zParse the input lines from a robots.txt file. We allow that a user-agent: line is not preceded by one or more blank lines. rr#N:z user-agentZdisallowFZallowT)Entryr.findstripsplitlenlowerrrunquoter+r, rulelinesRuleLine)rlinesstater-lineirrrrRsJ           zRobotFileParser.parsecCs|jr dS|jrdStjjtjj|}tjjdd|j|j|j |j f}tjj |}|s\d}x"|j D]}|j |rd|j|SqdW|jr|jj|SdS)z=using the parsed robots.txt decide if useragent can fetch urlFTr /)r rrrrr8 urlunparserparamsqueryfragmentquoter applies_to allowancer )r useragentr parsed_urlr-rrr can_fetchs     zRobotFileParser.can_fetchcCsdjdd|jDS)Nr cSsg|]}t|dqS) )r).0r-rrr sz+RobotFileParser.__str__..)joinr )rrrr__str__szRobotFileParser.__str__N)r ) __name__ __module__ __qualname____doc__rrrrr$r.rrIrNrrrrr s    3c@s(eZdZdZddZddZddZdS) r:zoA rule line is a single "Allow:" (allowance==True) or "Disallow:" (allowance==False) followed by a path.cCs*|dkr| rd}tjj||_||_dS)Nr T)rrrDrrF)rrrFrrrrszRuleLine.__init__cCs|jdkp|j|jS)Nr*)r startswith)rfilenamerrrrEszRuleLine.applies_tocCs|jr dp dd|jS)NZAllowZDisallowz: )rFr)rrrrrNszRuleLine.__str__N)rOrPrQrRrrErNrrrrr:sr:c@s0eZdZdZddZddZddZdd Zd S) r2z?An entry has one or more user-agents and zero or more rulelinescCsg|_g|_dS)N)r+r9)rrrrrszEntry.__init__cCsPg}x|jD]}|jd|dgq Wx |jD]}|jt|dgq,Wdj|S)Nz User-agent: rJr )r+extendr9rrM)rretagentr=rrrrNs   z Entry.__str__cCsF|jddj}x.|jD]$}|dkr*dS|j}||krdSqWdS)z2check if this entry applies to the specified agentr?rr*TF)r5r7r+)rrGrWrrrrEs zEntry.applies_tocCs$x|jD]}|j|r|jSqWdS)zZPreconditions: - our agent applies to this entry - filename is URL decodedT)r9rErF)rrTr=rrrrFs   zEntry.allowanceN)rOrPrQrRrrNrErFrrrrr2s  r2N) __future__rrrZfuture.builtinsrZfuture.backportsrZfuture.backports.urllibr_parser_request__all__objectr r:r2rrrrs