o 6ax@sdZdZddlZddlmmZddlm Z m Z d!ddZ ddZ d d Z d d Zd"ddZGdddZGdddZddZGdddeZGdddeZGdddeZGdddZd#dd ZdS)$zBA collection of functions designed to help I/O with ascii files. zrestructuredtext enN)asbytes asunicodecCs&t|tur|dur d}||}|S)aDecode bytes from binary input streams. Defaults to decoding from 'latin1'. That differs from the behavior of np.compat.asunicode that decodes from 'ascii'. Parameters ---------- line : str or bytes Line to be decoded. encoding : str Encoding used to decode `line`. Returns ------- decoded_line : unicode Unicode in Python 2, a str (unicode) in Python 3. Nlatin1)typebytesdecode)lineencodingr 4/usr/lib/python3/dist-packages/numpy/lib/_iotools.py _decode_line s  r c C(z|dWdSttfyYdSw)z2 Check whether obj behaves like a string. FT TypeError ValueErrorobjr r r _is_string_like&  rc Cr )z8 Check whether obj behaves like a bytes object. FTrrr r r _is_bytes_like1rrcCs(|jpdD] }||jdurdSqdS)a Returns whether one or several fields of a dtype are nested. Parameters ---------- ndtype : dtype Data-type of a structured array. Raises ------ AttributeError If `ndtype` does not have a `names` attribute. Examples -------- >>> dt = np.dtype([('name', 'S4'), ('x', float), ('y', float)]) >>> np.lib._iotools.has_nested_fields(dt) False r NTF)names)ndtypenamer r r has_nested_fields<s rFcCsf|j}|dur|r|jgtt|jS|jgSg}|D]}|j|}t|d|}||q|S)aX Unpack a structured data-type by collapsing nested fields and/or fields with a shape. Note that the field names are lost. Parameters ---------- ndtype : dtype The datatype to collapse flatten_base : bool, optional If True, transform a field with a shape into several fields. Default is False. Examples -------- >>> dt = np.dtype([('name', 'S4'), ('x', float), ('y', float), ... ('block', int, (2, 3))]) >>> np.lib._iotools.flatten_dtype(dt) [dtype('S4'), dtype('float64'), dtype('float64'), dtype('int64')] >>> np.lib._iotools.flatten_dtype(dt, flatten_base=True) [dtype('S4'), dtype('float64'), dtype('float64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64'), dtype('int64')] Nr) rbaseintnpprodshapefields flatten_dtypeextend)r flatten_basertypesfieldinfoflat_dtr r r r"Ws!  r"c@sFeZdZdZddZ  dddZd d Zd d Zd dZddZ dS) LineSplittera Object to split a string at a given delimiter or at given places. Parameters ---------- delimiter : str, int, or sequence of ints, optional If a string, character used to delimit consecutive fields. If an integer or a sequence of integers, width(s) of each field. comments : str, optional Character used to mark the beginning of a comment. Default is '#'. autostrip : bool, optional Whether to strip each individual field. Default is True. cs fddS)a Wrapper to strip each member of the output of `method`. Parameters ---------- method : function Function that takes a single argument and returns a sequence of strings. Returns ------- wrapped : function The result of wrapping `method`. `wrapped` takes a single input argument and returns a list of strings that are stripped of white-space. csdd|DS)NcSsg|]}|qSr )strip).0_r r r z...r )inputmethodr r sz(LineSplitter.autostrip..r )selfr1r r0r autostrips zLineSplitter.autostripN#TcCst|}t|}||_|dust|tr|pd}|j}n8t|drA|j}tdgt |}ddt |dd|ddD}nt |rN|j t |}}n|jd}}||_ |r`|||_n||_||_dS)N__iter__rcSsg|] \}}t||qSr slice)r+ijr r r r-z)LineSplitter.__init__..)r comments isinstancestr_delimited_splitterhasattr_variablewidth_splitterrcumsumlistzipr_fixedwidth_splitter delimiterr4 _handymanr )r3rHr>r4r rIidxr r r __init__s& &   zLineSplitter.__init__cCs8|jdur ||jd}|d}|sgS||jS)z2Chop off comments, strip, and split at delimiter. Nrz )r>splitr*rHr3rr r r rAs   z LineSplitter._delimited_splittercsb|jdur |jddsgS|jfddtdtD}fdd|DS)Nrz csg|] }t||qSr r7r+r9)fixedr r r-r;z5LineSplitter._fixedwidth_splitter..cg|]}|qSr r r+srr r r-r.)r>rLr*rHrangelenr3rslicesr )rOrr rGs  z!LineSplitter._fixedwidth_splittercs:|jdur |jdsgS|j}fdd|DS)NrcrPr r rQrSr r r-r.z8LineSplitter._variablewidth_splitter..)r>rLrHrVr rSr rCs z$LineSplitter._variablewidth_splittercCs|t||jSN)rIr r rMr r r __call__szLineSplitter.__call__)Nr5TN) __name__ __module__ __qualname____doc__r4rKrArGrCrYr r r r r)s   r)c@sBeZdZdZgdZedZ  d ddZdd d Zdd d Z dS) NameValidatora" Object to validate a list of strings to use as field names. The strings are stripped of any non alphanumeric character, and spaces are replaced by '_'. During instantiation, the user can define a list of names to exclude, as well as a list of invalid characters. Names in the exclusion list are appended a '_' character. Once an instance has been created, it can be called with a list of names, and a list of valid names will be created. The `__call__` method accepts an optional keyword "default" that sets the default name in case of ambiguity. By default this is 'f', so that names will default to `f0`, `f1`, etc. Parameters ---------- excludelist : sequence, optional A list of names to exclude. This list is appended to the default list ['return', 'file', 'print']. Excluded names are appended an underscore: for example, `file` becomes `file_` if supplied. deletechars : str, optional A string combining invalid characters that must be deleted from the names. case_sensitive : {True, False, 'upper', 'lower'}, optional * If True, field names are case-sensitive. * If False or 'upper', field names are converted to upper case. * If 'lower', field names are converted to lower case. The default value is True. replace_space : '_', optional Character(s) used in replacement of white spaces. Notes ----- Calling an instance of `NameValidator` is the same as calling its method `validate`. Examples -------- >>> validator = np.lib._iotools.NameValidator() >>> validator(['file', 'field2', 'with space', 'CaSe']) ('file_', 'field2', 'with_space', 'CaSe') >>> validator = np.lib._iotools.NameValidator(excludelist=['excl'], ... deletechars='q', ... case_sensitive=False) >>> validator(['excl', 'field2', 'no_q', 'with space', 'CaSe']) ('EXCL', 'FIELD2', 'NO_Q', 'WITH_SPACE', 'CASE') )returnfileprintz~!@#$%^&*()-=+~\|]}[{';: /?.>,.FucS|SrX)upperrcr r r r2/lcSrfrX)lowerrcr r r r21rhz%unrecognized case_sensitive value %s.) r#defaultexcludelist excludelistdefaultdeletecharssetadd deletecharscase_converter startswithr replace_space)r3rlrpcase_sensitiversdeletemsgr r r rKs$       zNameValidator.__init__f%ic s`|dur |dur dSg}t|tr|g}|dur6t|}||kr,t|dg||}n ||kr6|d|}|j|j}|j}|j}g}t} d} |D]`} ||  } |r[| d|} d fdd| D} | dkr|| } | |vr| d7} || } | |vss| d7} n| |vr| d7} | | d} | dkr| | d | n| | | d| | <qKt|S) a Validate a list of strings as field names for a structured array. Parameters ---------- names : sequence of str Strings to be validated. defaultfmt : str, optional Default format string, used if validating a given string reduces its length to zero. nbfields : integer, optional Final number of validated names, used to expand or shrink the initial list of names. Returns ------- validatednames : list of str The list of validated field names. Notes ----- A `NameValidator` instance can be called directly, which is the same as calling `validate`. For examples, see `NameValidator`. Nrr csg|]}|vr|qSr r )r+crpr r r-msz*NameValidator.validate..r=r,z_%d)r?r@rUrErprlrqrsdictr*replacejoingetappendtuple) r3r defaultfmtnbfieldsnbnamesrlrqrsvalidatednamesseennbemptyitemcntr rzr validate8sN       zNameValidator.validatecCs|j|||dS)N)rr)r)r3rrrr r r rY~szNameValidator.__call__)NNNr,rwN) rZr[r\r]rkrnrmrKrrYr r r r r^s3 Fr^cCs(|}|dkr dS|dkrdStd)a Tries to transform a string supposed to represent a boolean to a boolean. Parameters ---------- value : str The string that is transformed to a boolean. Returns ------- boolval : bool The boolean representation of `value`. Raises ------ ValueError If the string is not 'True' or 'False' (case independent) Examples -------- >>> np.lib._iotools.str2bool('TRUE') True >>> np.lib._iotools.str2bool('false') False TRUETFALSEFzInvalid boolean)rgr)valuer r r str2bools rc@eZdZdZdS)ConverterErrorzR Exception raised when an error occurs in a converter for string values. NrZr[r\r]r r r r rrc@r)ConverterLockErrorzR Exception raised when an attempt is made to upgrade a locked converter. Nrr r r r rrrc@r)ConversionWarningz Warning issued when a string converter has a problem. Notes ----- In `genfromtxt` a `ConversionWarning` is issued if raising exceptions is explicitly suppressed with the "invalid_raise" keyword. Nrr r r r rs rc @sNeZdZdZejedfejedfgZ e ejj e ej j kr)e ej edfe ejeejfejeejdfejejejfejedfejeejfejeejdfejedfejedfgeddZedd Zed d Zed"d dZ eddZ! d#ddZ"ddZ#ddZ$ddZ%ddZ&ddZ'ddZ( d$d d!Z)d S)%StringConverterab Factory class for function transforming a string into another object (int, float). After initialization, an instance can be called to transform a string into another object. If the string is recognized as representing a missing value, a default value is returned. Attributes ---------- func : function Function used for the conversion. default : any Default value to return when the input corresponds to a missing value. type : type Type of the output. _status : int Integer representing the order of the conversion. _mapper : sequence of tuples Sequence of tuples (dtype, function, default value) to evaluate in order. _locked : bool Holds `locked` parameter. Parameters ---------- dtype_or_func : {None, dtype, function}, optional If a `dtype`, specifies the input data type, used to define a basic function and a default value for missing data. For example, when `dtype` is float, the `func` attribute is set to `float` and the default value to `np.nan`. If a function, this function is used to convert a string to another object. In this case, it is recommended to give an associated default value as input. default : any, optional Value to return by default, that is, when the string to be converted is flagged as missing. If not given, `StringConverter` tries to supply a reasonable default value. missing_values : {None, sequence of str}, optional ``None`` or sequence of strings indicating a missing value. If ``None`` then missing values are indicated by empty entries. The default is ``None``. locked : bool, optional Whether the StringConverter should be locked to prevent automatic upgrade or not. Default is False. Fr<yz???cCs t|jS)z(Returns the dtype of the input variable.)rarraydtypeclsvalr r r _getdtype s zStringConverter._getdtypecCst|jjS)z4Returns the type of the dtype of the input variable.)rrrrrr r r _getsubdtypeszStringConverter._getsubdtypecCs|jtjkr|S|jS)z9Returns dtype for datetime64 and type of dtype otherwise.)rr datetime64)rrr r r _dtypeortypes zStringConverter._dtypeortypeNcCst|dr|jd||||fdSt|dret|dttfr0|D] }|jd|q$dS|dur>> import dateutil.parser >>> import datetime >>> dateparser = dateutil.parser.parse >>> defaultdate = datetime.date(2000, 1, 1) >>> StringConverter.upgrade_mapper(dateparser, default=defaultdate) rYr<Nr6r) rB_mapperinsertrr?rrErUrrF)rfuncdefaultr,fctdftr r r upgrade_mappers   zStringConverter.upgrade_mappercCstt|jD]\}\}}}|j|kr||||ffSqt|jD]\}\}}}t|j|r7||||ffSq trX) enumeraterrr issubdtype LookupError)rrr9deftyper default_defr r r _find_map_entryHs zStringConverter._find_map_entryc Cst||_|durt|_d|_|pd|_td}nz d|_t|}Wn7ty[t |ds:d}t|t |||_|durTz|d}Wn t ySd}Ynw| |}Ynwz | |\|_\}}} Wnty||_|jd\}}}d|_Yn w|dur| |_n||_|jdur||_|j|jddkrt|j tjrtj|_nt|j tjrtj|_nd d |_|durd h|_nt|tr|d }tt|d g|_|j|_|||_ d|_||_dS) NrFboolrYzOThe input argument `dtype` is neither a function nor a dtype (got '%s' instead)0r<r=cSs tt|SrX)rfloatrcr r r r2s z*StringConverter.__init__..r,)r_lockedrr_statusrrrrrBrrrrrr issubclassuint64int64missing_valuesr?r@rLrnrE _strict_call_callingfunctionr_checked_initial_default) r3 dtype_or_funcrrlockedrerrmsgr,rrr r r rKVsb                 zStringConverter.__init__cCs&z||WSty|jYSwrX)rrrr3rr r r _loose_calls    zStringConverter._loose_callcCsz#||}|jtur!z tj||jdW|WSty tw|WStyB||jvr<|j s7d|_ |j YStd|w)N)rFzCannot convert string '%s') rrrrr OverflowErrorrr*rrrr)r3r new_valuer r r rs"      zStringConverter._strict_callcCs ||SrX)rrr r r rYs zStringConverter.__call__cCs|jr d}t|t|j}|j}||krd}t|||dkr%|d7}|j|\|_|_}||_|jdur=|j|_ dS||_ dS)Nz*Converter is locked and cannot be upgradedz*Could not find a valid conversion functionr=) rrrUrrrrrrr)r3r _statusmaxrrr r r _do_upgrades     zStringConverter._do_upgradecCs8d|_z||WSty|||YSw)a Find the best converter for a given string, and return the result. The supplied string `value` is converted by testing different converters in order. First the `func` method of the `StringConverter` instance is tried, if this fails other available converters are tried. The order in which these other converters are tried is determined by the `_status` attribute of the instance. Parameters ---------- value : str The string to convert. Returns ------- out : any The result of converting `value` with the appropriate converter. T)rrrrupgraderr r r rs  zStringConverter.upgradecCs\d|_t|ds |f}|j}z |D]}||qWdSty-|||YdSw)NTr6)rrBrrr iterupgrade)r3rr_mr r r rs   zStringConverter.iterupgraderc Cs||_||_|dur||_||||_nz||pd}Wn ttfy,d}Ynw||||_|dur@t|_ dSt |sH|g}t dd|DsUtd|j |dS)a Set StringConverter attributes directly. Parameters ---------- func : function Conversion function. default : any, optional Value to return by default, that is, when the string to be converted is flagged as missing. If not given, `StringConverter` tries to supply a reasonable default value. testing_value : str, optional A string representing a standard input value of the converter. This string is used to help defining a reasonable default value. missing_values : {sequence of str, None}, optional Sequence of strings indicating a missing value. If ``None``, then the existing `missing_values` are cleared. The default is `''`. locked : bool, optional Whether the StringConverter should be locked to prevent automatic upgrade or not. Default is False. Notes ----- `update` takes the same parameters as the constructor of `StringConverter`, except that `func` does not accept a `dtype` whereas `dtype_or_func` in the constructor does. N1css|]}t|tVqdSrX)r?r@)r+vr r r 2sz)StringConverter.update..z)missing_values must be strings or unicode)rrrrrrrrrnrriterableallupdate)r3rr testing_valuerrtesterr r r rs$  zStringConverter.updaterX)NNNF)NNrF)*rZr[r\r]nxbool_rint_rrritemsizerrr#float64rnan complex128complex longdoubleintegerfloatingcomplexfloatingunicode_rstring_r classmethodrrrrrrKrrrYrrrrr r r r rsL /         (   I rrwc Kszt|}Wn;tyBtd i|}t|}|dur%dgt|}n t|tr/|d}||||d}tt||d}Y|Sw|durtd i|}t|trX|d}|j duryt |j gt|}|||d}tt t ||}|S||t|j |d|_ |S|j durtd i|}t ddtt|j D}|j |kr|d kr|dgt|j |d|_ |S||j |d|_ |S) a6 Convenience function to create a `np.dtype` object. The function processes the input `dtype` and matches it with the given names. Parameters ---------- ndtype : var Definition of the dtype. Can be any string or dictionary recognized by the `np.dtype` function, or a sequence of types. names : str or sequence, optional Sequence of strings to use as field names for a structured dtype. For convenience, `names` can be a string of a comma-separated list of names. defaultfmt : str, optional Format string used to define missing names, such as ``"f%i"`` (default) or ``"fields_%02i"``. validationargs : optional A series of optional arguments used to initialize a `NameValidator`. Examples -------- >>> np.lib._iotools.easy_dtype(float) dtype('float64') >>> np.lib._iotools.easy_dtype("i4, f8") dtype([('f0', '>> np.lib._iotools.easy_dtype("i4, f8", defaultfmt="field_%03i") dtype([('field_000', '>> np.lib._iotools.easy_dtype((int, float, float), names="a,b,c") dtype([('a', '>> np.lib._iotools.easy_dtype(float, names="a,b,c") dtype([('a', '.rwr )rrrr^rUr?r@rLr{rrrrErFrT)rrrvalidationargsrrrnumbered_namesr r r easy_dtype7sH&          rrX)F)Nrw)r] __docformat__numpyrnumpy.core.numericcorenumericr numpy.compatrrr rrrr"r)r^r Exceptionrr UserWarningrrrr r r r s*   /`$ v