o }5h$@s\dZddlZddlZddlZdgZeddZGdddZGdddZ Gd d d Z dS) a% robotparser.py Copyright (C) 2000 Bastian Kleineidam You can choose between two licenses when using this package: 1) GNU GPLv2 2) PSF license for Python 2.2 The robots.txt Exclusion Protocol is implemented as specified in http://www.robotstxt.org/norobots-rfc.txt NRobotFileParser RequestRatezrequests secondsc@sreZdZdZdddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZdS)rzs This class provides a set of methods to read, parse and answer questions about a single robots.txt file. cCs2g|_g|_d|_d|_d|_||d|_dS)NFr)entriessitemaps default_entry disallow_all allow_allset_url last_checkedselfurlr)/usr/lib/python3.10/urllib/robotparser.py__init__s  zRobotFileParser.__init__cCs|jS)zReturns the time the robots.txt file was last fetched. This is useful for long-running web spiders that need to check for new robots.txt files periodically. )r r rrrmtime%szRobotFileParser.mtimecCsddl}||_dS)zYSets the time the robots.txt file was last fetched to the current time. rN)timer )r rrrrmodified.szRobotFileParser.modifiedcCs&||_tj|dd\|_|_dS)z,Sets the URL referring to a robots.txt file.N)rurllibparseurlparsehostpathr rrrr 6s zRobotFileParser.set_urlc Csz tj|j}Wn@tjjyI}z2|jdvrd|_n|jdkr0|jdkr>d|_WYd}~dSWYd}~dSWYd}~dSWYd}~dSd}~ww| }| | d dS)z4Reads the robots.txt URL and feeds it to the parser.)iiTiiNzutf-8) rrequesturlopenrerror HTTPErrorcoderr readrdecode splitlines)r ferrrawrrrr";s zRobotFileParser.readcCs2d|jvr|jdur||_dSdS|j|dSN*) useragentsrrappend)r entryrrr _add_entryHs   zRobotFileParser._add_entrycCsJd}t}||D] }|s(|dkrt}d}n|dkr(||t}d}|d}|dkr7|d|}|}|s>q |dd}t|dkr|d|d<tj |d|d<|ddkr~|dkrs||t}|j |dd}q |ddkr|dkr|j t|dd d}q |dd kr|dkr|j t|dd d}q |dd kr|dkr|drt|d|_d}q |dd kr|dkr|dd}t|dkr|dr|drtt|dt|d|_d}q |ddkr|j |dq |dkr#||dSdS)zParse the input lines from a robots.txt file. We allow that a user-agent: line is not preceded by one or more blank lines. rr#N:z user-agentdisallowFallowTz crawl-delayz request-rate/sitemap)Entryrr-findstripsplitlenlowerrrunquoter*r+ rulelinesRuleLineisdigitintdelayrreq_rater)r linesstater,lineinumbersrrrrQsv              zRobotFileParser.parsecCs|jrdS|jr dS|jsdStjtj|}tjdd|j|j |j |j f}tj |}|s3d}|j D]}||rD||Sq6|jrN|j|SdS)z=using the parsed robots.txt decide if useragent can fetch urlFTrr3)rr r rrrr; urlunparserparamsqueryfragmentquoter applies_to allowancer)r useragentr parsed_urlr,rrr can_fetchs(     zRobotFileParser.can_fetchcC>|sdS|jD] }||r|jSq |jr|jjSdSN)rrrLr@rr rNr,rrr crawl_delay   zRobotFileParser.crawl_delaycCrQrR)rrrLrArrSrrr request_raterUzRobotFileParser.request_ratecCs|jsdS|jSrR)rrrrr site_mapsszRobotFileParser.site_mapscCs,|j}|jdur||jg}dtt|S)Nz )rrjoinmapstr)r rrrr__str__s  zRobotFileParser.__str__N)r)__name__ __module__ __qualname____doc__rrrr r"r-rrPrTrVrWr[rrrrrs     I  c@s(eZdZdZddZddZddZdS) r=zoA rule line is a single "Allow:" (allowance==True) or "Disallow:" (allowance==False) followed by a path.cCs<|dkr|sd}tjtj|}tj||_||_dS)NrT)rrrGrrKrrM)r rrMrrrrs  zRuleLine.__init__cCs|jdkp ||jSr()r startswith)r filenamerrrrLszRuleLine.applies_tocCs|jrdndd|jS)NAllowDisallowz: )rMrrrrrr[szRuleLine.__str__N)r\r]r^r_rrLr[rrrrr=s  r=c@s0eZdZdZddZddZddZdd Zd S) r5z?An entry has one or more user-agents and zero or more rulelinescCsg|_g|_d|_d|_dSrR)r*r<r@rArrrrrs zEntry.__init__cCsg}|jD] }|d|q|jdur|d|j|jdur3|j}|d|jd|j|tt|j d |S)Nz User-agent: z Crawl-delay: zRequest-rate: r3 ) r*r+r@rArequestssecondsextendrYrZr<rX)r retagentraterrrr[s    z Entry.__str__cCsF|dd}|jD]}|dkrdS|}||vr dSq dS)z2check if this entry applies to the specified agentr3rr)TF)r8r:r*)r rNrirrrrLs zEntry.applies_tocCs$|jD] }||r|jSqdS)zZPreconditions: - our agent applies to this entry - filename is URL decodedT)r<rLrM)r rarDrrrrM s   zEntry.allowanceN)r\r]r^r_rr[rLrMrrrrr5s  r5) r_ collections urllib.parserurllib.request__all__ namedtuplerrr=r5rrrrs  B