3 QfX @sdZddlmZddlZddlZddlZddlZddlZddlZy ddl Z Wne k rhddl Z YnXddl Z ddl Z ddlZddlZddlZddlZddlZddlmZddlmZmZmZddlmZmZddlmZdd lmZmZm Z dd lm!Z!dd l"m#Z#m$Z$dd l%m&Z&d Z'ddZ(Gddde)Z*ddZ+ddZ,d"ddZ-ej.Z/GdddeZ0Gddde1Z2Gddde1Z3GdddeZ4Gd d!d!eZ5dS)#zo A context object for caching a function's return value each time it is called with the same input arguments. )with_statementN)hashing) get_func_code get_func_name filter_args)format_signature format_call)open_py_source)Logger format_timepformat) numpy_pickle)mkdirp rm_subdirs) _basestringz # first line:cCsL|jtr@|jd}t|dttd}dj|dd}nd}||fS)zZ Extract the first line information from the function code text if available.  rNr) startswithFIRST_LINE_TEXTsplitintlenjoin) func_code first_linerT/home/psgendb/BIRCHDEV/pkg/SPAdes-3.15.4/linux-x86_64/share/spades/joblib3/memory.pyextract_first_line8s   rc@seZdZdZdS)JobLibCollisionWarningzF Warn that there might be a collision between names of functions. N)__name__ __module__ __qualname____doc__rrrrrEsrcCs"t|\}}|j|tjj|S)ziCompute the part of part associated with a function. See code of_cache_key_to_dir() for details )rappendospathr)funcmodulesfuncnamerrr_get_func_fullnameJs  r*cCsH|g}t|tr|j|n|jt||dk r<|j|tjj|S)zCompute directory associated with a given cache key. func can be a function or a string as returned by _get_func_fullname(). N) isinstancerr$r*r%r&r)cachedirr' argument_hashpartsrrr_cache_key_to_dirTs   r/c Cs|dkrd}yJ|dk rHdjdd|djD}dtjj||f}n tjj|}Wntk rjYnX|dk rd ttj|}nd}|d krtd |t |fntd |t ||ftjj|d } tjj | std| t j | |dS)zLoad output of a computation.rNz, cSsg|]\}}d||fqS)z%s=%sr).0namevaluerrr ksz _load_output.. input_argsz%s(%s)z% 16s z[Memory]%s: Loading %s...z[Memory]%s: Loading %s from %sz output.pklzHNon-existing cache value (may have been cleared). File %s does not exist) mmap_mode) ritemsr%r&basenameKeyErrorr timeprintstrisfilerload) output_dir func_name timestampmetadatar7verboseZ signatureargstfilenamerrr _load_outputds0    rHc@s:eZdZdZdddZddZdd Zd d Zd d ZdS)MemorizedResultaObject representing a cached value. Attributes ---------- cachedir: string path to root of joblib cache func: function or string function whose output is cached. The string case is intended only for instanciation based on the output of repr() on another instance. (namely eval(repr(memorized_instance)) works). argument_hash: string hash of the function arguments mmap_mode: {None, 'r+', 'r', 'w+', 'c'} The memmapping mode used when loading from cache numpy arrays. See numpy.load for the meaning of the different values. verbose: int verbosity level (0 means no message) timestamp, metadata: string for internal use only Nrc Cstj|t|tr||_n t||_||_||_||_t ||j||_ |dk rX||_ nFi|_ y2t t jj|j dd}tj||_ WdQRXWn YnX|j jdd|_||_||_dS)Nz metadata.jsonrbduration)r __init__r+rr'r*r-r,r7r/ _output_dirrCopenr%r&rjsonr?getrKrDrB) selfr,r'r-r7rDrBrCfrrrrLs*   zMemorizedResult.__init__cCs$t|jt|j|j|j|j|jdS)z$Read value from cache and return it.)rBrCr7rD)rHrMr*r'rBrCr7rD)rQrrrrPszMemorizedResult.getcCstj|jdddS)zClear value from cacheT) ignore_errorsN)shutilrmtreerM)rQrrrclearszMemorizedResult.clearcCsdj|jj|j|j|jdS)NzS{class_name}(cachedir="{cachedir}", func="{func}", argument_hash="{argument_hash}")) class_namer,r'r-)format __class__r r,r'r-)rQrrr__repr__s zMemorizedResult.__repr__cCs|j|j|j|jfd|jifS)Nr7)rYr,r'r-r7)rQrrr __reduce__szMemorizedResult.__reduce__)NrNN) r r!r"r#rLrPrVrZr[rrrrrIs   rIc@sDeZdZdZdZddZddZdd Zd d Zd d Z ddZ dS)NotMemorizedResultzxClass representing an arbitrary value. This class is a replacement for MemorizedResult when there is no cache. r3validcCs||_d|_dS)NT)r3r])rQr3rrrrLszNotMemorizedResult.__init__cCs|jr |jStddS)NzNo value stored.)r]r3r:)rQrrrrPszNotMemorizedResult.getcCsd|_d|_dS)NF)r]r3)rQrrrrVszNotMemorizedResult.clearcCs.|jrdj|jjt|jdS|jjdSdS)Nz{class_name}({value}))rWr3z with no value)r]rXrYr r r3)rQrrrrZs zNotMemorizedResult.__repr__cCs|j|jdS)N)r]r3)r]r3)rQrrr __getstate__szNotMemorizedResult.__getstate__cCs|d|_|d|_dS)Nr]r3)r]r3)rQstaterrr __setstate__s zNotMemorizedResult.__setstate__N)r3r]) r r!r"r# __slots__rLrPrVrZr^r`rrrrr\s r\c@sBeZdZdZddZddZddZdd Zd d Zdd dZ dS)NotMemorizedFuncaNo-op object decorating a function. This class replaces MemorizedFunc when there is no cache. It provides an identical API but does not write anything on disk. Attributes ---------- func: callable Original undecorated function. cCs ||_dS)N)r')rQr'rrrrLszNotMemorizedFunc.__init__cOs |j||S)N)r')rQrEkwargsrrr__call__szNotMemorizedFunc.__call__cOst|j||S)N)r\r')rQrErcrrrcall_and_shelvesz NotMemorizedFunc.call_and_shelvecCs|j|jffS)N)rYr')rQrrrr[ szNotMemorizedFunc.__reduce__cCsd|jj|jfS)Nz %s(func=%s))rYr r')rQrrrrZ#szNotMemorizedFunc.__repr__TcCsdS)Nr)rQwarnrrrrV)szNotMemorizedFunc.clearN)T) r r!r"r#rLrdrer[rZrVrrrrrb s rbc@seZdZdZd.ddZddZd d Zd d Zd dZddZ ddZ ddZ ddZ e Z d/ddZddZddZd0dd Zd1d!d"Zd#d$Zd%d&Zd2d(d)Zd*d+Zd,d-ZdS)3 MemorizedFunca Callable object decorating a function for caching its return value each time it is called. All values are cached on the filesystem, in a deep directory structure. Methods are provided to inspect the cache or clean it. Attributes ---------- func: callable The original, undecorated, function. cachedir: string Path to the base cache directory of the memory context. ignore: list or None List of variable names to ignore when choosing whether to recompute. mmap_mode: {None, 'r+', 'r', 'w+', 'c'} The memmapping mode used when loading from cache numpy arrays. See numpy.load for the meaning of the different values. compress: boolean, or integer Whether to zip the stored data on disk. If an integer is given, it should be between 1 and 9, and sets the amount of compression. Note that compressed arrays cannot be read by memmapping. verbose: int, optional The verbosity flag, controls messages that are issued as the function is evaluated. NFrc Cstj|||_||_|dkr"g}||_||_||_||_|rV|jdk rVtj ddd|dkrft j }||_ t |jyt j||Wn YnXtj|rtjj|}|jddd}tjdd |}n|j}d ||_dS) a{ Parameters ---------- func: callable The function to decorate cachedir: string The path of the base directory to use as a data store ignore: list or None List of variable names to ignore. mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional The memmapping mode used when loading from cache numpy arrays. See numpy.load for the meaning of the arguments. compress : boolean, or integer Whether to zip the stored data on disk. If an integer is given, it should be between 1 and 9, and sets the amount of compression. Note that compressed arrays cannot be read by memmapping. verbose: int, optional Verbosity flag, controls the debug messages that are issued as functions are evaluated. The higher, the more verbose timestamp: float, optional The reference time from which times in tracing messages are reported. Nz&Compressed results cannot be memmapped) stacklevelrz rz.r0zMemoized version of %s)r rLr7r'ignore_verboser,compresswarningsrfr;rBr functoolsupdate_wrapperinspectZ isfunctionpydocZTextDocdocumentreplaceresubr#) rQr'r,rjr7rlrDrBdocrrrrLWs4   zMemorizedFunc.__init__c Csn|j||\}}d}|jddo*tjj|s|jdkrXt|j\}}|jd|||f|j ||\}}|j dk rt |t |j|j |j |jd}nytj} t |t |j|j ||j |jd}|jdkrtj| } t|j\}}d|t| f} ttd d t| d | WnPtk rb|jd ||tjftj|d d|j ||\}}d}YnX|||fS)aCall wrapped function and cache result, or read cache if available. This function returns the wrapped function output and some metadata. Returns ------- output: value or tuple what is returned by wrapped function argument_hash: string hash of function arguments metadata: dict some metadata about wrapped function call (see _persist_input()) N)rir6z3Computing func %s, argument hash %s in directory %s)rBr7rD)rBrCr7rDz%s cache loaded - %srP_zrA) riorNwritesys version_infohasattrr'r rAr_FUNCTION_HASHES TypeError)rQrGrrrZis_named_callable func_hashrrr_write_func_codes    zMemorizedFunc._write_func_coderhc%Cs6y(|jtkr&|j}|t|jkr&dSWntk r<YnXt|j\}}}|j}tjj|d}y.t j |dd}t |j \} } WdQRXWn"t k r|j|||dSX| |krdSt|jddd\} } | |kodkns| d kr,|dksd | ||f} n| } tjtd | |d | |k r|dk rd}tjj|rt|jdd \} } t|jd}t|&}|j| d| d|d}WdQRXdj|}|j| jk}n |jd}|rtjtd| || | ||f|d |jdkr&t|jdd \} } |jd| |f|jdddS)z stacklevel is the depth a which this function is called, to issue useful warnings to the user. Tz func_code.pyzUTF-8)rNF) resolv_aliasZwin_charactersrzz %s (%s:%i)z/Cannot detect name collisions for function '%s')ri)rrr0z sz/MemorizedFunc._persist_input..)rKr5z metadata.jsonrNa~Persisting input arguments took %.2fs to run. If this happens often in your code, it can cause performance problems (results will be correct in all cases). The reason for this is probably some large input arguments for a wrapped function (e.g. large strings). THIS IS A JOBLIB ISSUE. If you can, kindly provide the joblib's team with an example so that they can fix the problem.)ri)r;rr'rjdictr8rrNr%r&rrOrrmrf) rQr@rKrErcZthis_duration_limitrZ argument_dictZ input_reprrCrRZ this_durationrrrrs     zMemorizedFunc._persist_inputcCs*tjdtt|t|j|j|j|jdS)ze Read the results of a previous calculation from the directory it was cached in. zMemorizedFunc.load_output is deprecated and will be removed in a future version of joblib. A MemorizedResult provides similar features)rBr7rD) rmrfrrHr*r'rBr7rk)rQr@rrr load_outputs  zMemorizedFunc.load_outputcCsd|jj|jt|jfS)Nz%s(func=%s, cachedir=%s))rYr r'rr,)rQrrrrZszMemorizedFunc.__repr__)NNFrN)T)rh)T)r)r r!r"r#rLrrerdr[rr rrzZget_output_dirrrrr{rVr}rrrrZrrrrrg1s,! :;    X  5rgc@sFeZdZdZdddZdddZdd d Zd d ZddZddZ dS)Memoryz A context object for caching a function's return value each time it is called with the same input arguments. All values are cached on the filesystem, in a deep directory structure. see :ref:`memory_reference` NFrcCsntj|||_||_tj|_||_|r@|dk r@tjddd|dkrPd|_ nt j j |d|_ t |j dS)a Parameters ---------- cachedir: string or None The path of the base directory to use as a data store or None. If None is given, no caching is done and the Memory object is completely transparent. mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional The memmapping mode used when loading from cache numpy arrays. See numpy.load for the meaning of the arguments. compress: boolean, or integer Whether to zip the stored data on disk. If an integer is given, it should be between 1 and 9, and sets the amount of compression. Note that compressed arrays cannot be read by memmapping. verbose: int, optional Verbosity flag, controls the debug messages that are issued as functions are evaluated. Nz&Compressed results cannot be memmappedrh)riZjoblib)r rLrkr7r;rBrlrmrfr,r%r&rr)rQr,r7rlrDrrrrLs   zMemory.__init__c Csv|dkrtj|j|||dS|jdkr.t|S|dkr<|j}|dkrJ|j}t|trZ|j }t||j|||j ||j dS)a{ Decorates the given function func to only compute its return value for input arguments not cached on disk. Parameters ---------- func: callable, optional The function to be decorated ignore: list of strings A list of arguments name to ignore in the hashing verbose: integer, optional The verbosity mode of the function. By default that of the memory object is used. mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional The memmapping mode used when loading from cache numpy arrays. See numpy.load for the meaning of the arguments. By default that of the memory object is used. Returns ------- decorated_func: MemorizedFunc object The returned object is a MemorizedFunc object, that is callable (behaves like a function), but offers extra methods for cache lookup and management. See the documentation for :class:`joblib.memory.MemorizedFunc`. N)rjrDr7F)r,r7rjrlrDrB) rnpartialcacher,rbrkr7r+rgr'rlrB)rQr'rjrDr7rrrrCs"    z Memory.cacheTcCs&|r|jd|jdk r"t|jdS)z- Erase the complete cache directory. zFlushing completely the cacheN)rfr,r)rQrfrrrrVrs  z Memory.clearcOs$|jdkr|||S|j|||S)a Eval function func with arguments `*args` and `**kwargs`, in the context of the memory. This method works similarly to the builtin `apply`, except that the function is called only if the cache is not up to date. N)r,r)rQr'rErcrrrevalzs  z Memory.evalcCsd|jjt|jfS)Nz%s(cachedir=%s))rYr rr,)rQrrrrZszMemory.__repr__cCs4|jdk r|jddnd}|j||j|j|jffS)z We don't store the timestamp when pickling, to avoid the hash depending from it. In addition, when unpickling, we run the __init__ Ni)r,rYr7rlrk)rQr,rrrr[szMemory.__reduce__)NFr)NNNF)T) r r!r"r#rLrrVrrZr[rrrrrs $ . r)NNNr)6r#Z __future__rr%rTr;rqrtrZcPicklepickle ImportErrorrnrrmrprOweakrefrr0rZ func_inspectrrrrr Z_memory_helpersr loggerr r r rZdiskrrZ_compatrrr UserWarningrr*r/rHWeakKeyDictionaryrrIobjectr\rbrgrrrrrsT         %Q*&d