# -*- coding: utf-8 -*-
"""
python -c "import utool, doctest; print(doctest.testmod(utool.util_path))"
This module becomes nav
"""
from __future__ import absolute_import, division, print_function, unicode_literals
from six.moves import zip, filter, filterfalse, map, range
import six
from os.path import (join, basename, relpath, normpath, split, isdir, isfile,
exists, islink, ismount, dirname, splitext, realpath,
splitdrive, commonprefix, expanduser)
import os
import re
import sys
import shutil
import fnmatch
import warnings
import itertools
from utool.util_regex import extend_regex
from utool import util_dbg
from utool import util_progress
from utool._internal import meta_util_path
from utool import util_inject
from utool import util_arg
from utool._internal.meta_util_arg import NO_ASSERTS, VERBOSE, VERYVERBOSE, QUIET
print, print_, printDBG, rrr, profile = util_inject.inject(__name__, '[util_path]')
PRINT_CALLER = util_arg.get_argflag('--print-caller') # FIXME: name
__IMG_EXTS = ['.jpg', '.jpeg', '.png', '.tif', '.tiff', '.ppm']
__LOWER_EXTS = list(ext.lower() for ext in __IMG_EXTS)
__UPPER_EXTS = list(ext.upper() for ext in __IMG_EXTS)
IMG_EXTENSIONS = set(__LOWER_EXTS + __UPPER_EXTS)
[docs]def newcd(path):
""" DEPRICATE """
cwd = os.getcwd()
os.chdir(path)
return cwd
unixpath = meta_util_path.unixpath
truepath = meta_util_path.truepath
unixjoin = meta_util_path.unixjoin
[docs]def ensure_ext(fname, ext):
r"""
Args:
fname (str): file name
ext (str):
Returns:
str: fname - file name
CommandLine:
python -m utool.util_path --exec-ensure_ext --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> print(ensure_ext('foo', '.bar'))
>>> print(ensure_ext('foo.bar', '.bar'))
"""
fname_, ext_ = splitext(fname)
if ext != ext_:
fname = fname_ + ext_ + ext
return fname
[docs]def relpath_unix(path, otherpath):
return relpath(path, otherpath).replace('\\', '/')
[docs]def truepath_relative(path, otherpath=None):
""" Normalizes and returns absolute path with so specs
Args:
path (str): path to file or directory
otherpath (None): (default = None)
Returns:
str: path_
CommandLine:
python -m utool.util_path --exec-truepath_relative --show
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> path = 'C:/foobar/foobiz'
>>> otherpath = 'C:/foobar'
>>> path_ = truepath_relative(path, otherpath)
>>> result = ('path_ = %s' % (ut.repr2(path_),))
>>> print(result)
path_ = 'foobiz'
"""
if otherpath is None:
otherpath = os.getcwd()
otherpath = truepath(otherpath)
path_ = normpath(relpath(path, otherpath))
return path_
[docs]def tail(fpath, n=2, trailing=True):
""" Alias for path_ndir_split """
return path_ndir_split(fpath, n=n, trailing=trailing)
[docs]def unexpanduser(path):
homedir = expanduser('~')
if path.startswith(homedir):
path = '~' + path[len(homedir):]
return path
[docs]def path_ndir_split(path_, n, force_unix=True, winroot='C:', trailing=True):
r"""
Shows only a little bit of the path. Up to the n bottom-level directories
TODO: rename to path_tail? ndir_split?
Returns:
(str) the trailing n paths of path.
CommandLine:
python3 -m utool.util_path --test-path_ndir_split
python3 -m utool --tf path_ndir_split
python -m utool --tf path_ndir_split
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> paths = [r'/usr/bin/local/foo/bar',
... r'C:/',
... #r'lonerel',
... #r'reldir/other',
... r'/ham',
... r'./eggs',
... r'/spam/eggs',
... r'C:\Program Files (x86)/foobar/bin',]
>>> N = 2
>>> iter_ = ut.iprod(paths, range(1, N + 1))
>>> force_unix = True
>>> tuplist = [(n, path_ndir_split(path_, n)) for path_, n in iter_]
>>> chunklist = list(ut.ichunks(tuplist, N))
>>> list_ = [['n=%r: %s' % (x, ut.reprfunc(y)) for x, y in chunk]
>>> for chunk in chunklist]
>>> line_list = [', '.join(strs) for strs in list_]
>>> result = '\n'.join(line_list)
>>> print(result)
n=1: '.../bar', n=2: '.../foo/bar'
n=1: 'C:/', n=2: 'C:/'
n=1: '.../ham', n=2: '/ham'
n=1: '.../eggs', n=2: './eggs'
n=1: '.../eggs', n=2: '.../spam/eggs'
n=1: '.../bin', n=2: '.../foobar/bin'
"""
if n is None:
return ensure_crossplat_path(path_)
if n == 0:
return ''
sep = '/' if force_unix else os.sep
ndirs_list = []
head = path_
reached_end = False
for nx in range(n):
#print('--')
#print('IN head = %r' % (head,))
head, tail = split(head)
#print('head = %r' % (head,))
#print('tail = %r' % (tail,))
if tail == '':
if head == '':
reached_end = True
break
else:
root = head if len(ndirs_list) == 0 else head.strip('\\/')
ndirs_list.append(root)
reached_end = True
break
else:
ndirs_list.append(tail)
#print('ndirs_list = %r' % (ndirs_list,))
if trailing and not reached_end:
head, tail = split(head)
#print('--')
#print('head = %r' % (head,))
#print('tail = %r' % (tail,))
if len(tail) == 0:
if len(head) == 0: # or head == '/':
reached_end = True
#if head == '/':
# reached_end = True
ndirs = sep.join(ndirs_list[::-1])
cplat_path = ensure_crossplat_path(ndirs)
#if trailing and not reached_end:
if trailing and not reached_end:
cplat_path = '.../' + cplat_path
#print('cplat_path = %r' % (cplat_path,))
#print('--')
return cplat_path
[docs]def remove_file(fpath, verbose=True, dryrun=False, ignore_errors=True, **kwargs):
""" Removes a file """
if dryrun:
if verbose:
print('[util_path] Dryrem %r' % fpath)
return
else:
try:
os.remove(fpath)
if verbose and not QUIET:
print('[util_path] Removed %r' % fpath)
except OSError:
print('[util_path.remove_file] Misrem %r' % fpath)
#warnings.warn('OSError: %s,\n Could not delete %s' % (str(e), fpath))
if not ignore_errors:
raise
return False
return True
[docs]def remove_dirs(dpath, dryrun=False, ignore_errors=True, quiet=QUIET, **kwargs):
r"""
Recursively removes a single directory (need to change function name)
DEPRICATE
Args:
dpath (str): directory path
dryrun (bool): (default = False)
ignore_errors (bool): (default = True)
quiet (bool): (default = False)
Returns:
bool: False
CommandLine:
python -m utool.util_path --test-remove_dirs
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> dpath = ut.ensure_app_resource_dir('utool', 'testremovedir')
>>> assert exists(dpath), 'nothing to remove'
>>> dryrun = False
>>> ignore_errors = True
>>> quiet = False
>>> flag = remove_dirs(dpath, dryrun, ignore_errors, quiet)
>>> result = ('flag = %s' % (flag,))
>>> print(result)
>>> assert not exists(dpath), 'did not remove dpath'
flag = True
"""
if not quiet:
print('[util_path] Removing directory: %r' % dpath)
if dryrun:
return False
try:
shutil.rmtree(dpath)
except OSError as e:
warnings.warn('OSError: %s,\n Could not delete %s' % (str(e), dpath))
if not ignore_errors:
raise
return False
return True
#import os
[docs]def augpath(path, augsuf='', augext='', augpref='', augdir=None, newext=None,
newfname=None, ensure=False):
"""
augments end of path before the extension.
augpath
Args:
path (str):
augsuf (str): augment filename before extension
Returns:
str: newpath
Example:
>>> from utool.util_path import * # NOQA
>>> path = 'somefile.txt'
>>> augsuf = '_aug'
>>> newpath = augpath(path, augsuf)
>>> result = str(newpath)
>>> print(result)
somefile_aug.txt
Example:
>>> from utool.util_path import * # NOQA
>>> path = 'somefile.txt'
>>> augsuf = '_aug2'
>>> newext = '.bak'
>>> augdir = 'backup'
>>> newpath = augpath(path, augsuf, newext=newext, augdir=augdir)
>>> result = str(newpath)
>>> print(result)
backup/somefile_aug2.bak
"""
# Breakup path
dpath, fname = split(path)
fname_noext, ext = splitext(fname)
if newfname is not None:
fname_noext = newfname
# Augment ext
if newext is None:
newext = ext
# Augment fname
new_fname = ''.join((augpref, fname_noext, augsuf, newext, augext))
# Augment dpath
if augdir is not None:
new_dpath = join(dpath, augdir)
if ensure:
# create new dir if needebe
ensuredir(new_dpath)
else:
new_dpath = dpath
# Recombine into new path
newpath = join(new_dpath, new_fname)
return newpath
[docs]def touch(fname, times=None, verbose=True):
r"""
Args:
fname (str)
times (None):
verbose (bool):
Example:
>>> from utool.util_path import * # NOQA
>>> fname = '?'
>>> times = None
>>> verbose = True
>>> result = touch(fname, times, verbose)
>>> print(result)
References:
http://stackoverflow.com/questions/1158076/implement-touch-using-python
"""
try:
if verbose:
print('[util_path] touching %r' % fname)
with open(fname, 'a'):
os.utime(fname, times)
except Exception as ex:
import utool
utool.printex(ex, 'touch %s' % fname)
raise
[docs]def remove_files_in_dir(dpath, fname_pattern_list='*', recursive=False,
verbose=VERBOSE, dryrun=False, ignore_errors=False,
**kwargs):
""" Removes files matching a pattern from a directory """
if isinstance(fname_pattern_list, six.string_types):
fname_pattern_list = [fname_pattern_list]
if not QUIET:
print('[util_path] Removing files:')
print(' * from dpath = %r ' % dpath)
print(' * with patterns = %r' % fname_pattern_list)
print(' * recursive = %r' % recursive)
num_removed, num_matched = (0, 0)
kwargs.update({
'dryrun': dryrun,
'verbose': verbose,
})
if not exists(dpath):
msg = ('!!! dir = %r does not exist!' % dpath)
if not QUIET:
print(msg)
warnings.warn(msg, category=UserWarning)
for root, dname_list, fname_list in os.walk(dpath):
for fname_pattern in fname_pattern_list:
for fname in fnmatch.filter(fname_list, fname_pattern):
num_matched += 1
num_removed += remove_file(join(root, fname),
ignore_errors=ignore_errors, **kwargs)
if not recursive:
break
print('[util_path] ... Removed %d/%d files' % (num_removed, num_matched))
return True
[docs]def delete(path, dryrun=False, recursive=True, verbose=VERBOSE,
print_exists=True, ignore_errors=True, **kwargs):
""" Removes a file or directory """
if not QUIET:
print('[util_path] Deleting path=%r' % path)
if not exists(path):
if print_exists and not QUIET:
msg = ('..does not exist!')
print(msg)
return False
rmargs = dict(dryrun=dryrun, recursive=recursive, verbose=verbose,
ignore_errors=ignore_errors, **kwargs)
if isdir(path):
flag = remove_files_in_dir(path, **rmargs)
flag = flag and remove_dirs(path, **rmargs)
elif isfile(path):
flag = remove_file(path, **rmargs)
else:
raise ValueError('Unknown type of path=%r' (path,))
if not QUIET:
print('[util_path] Finished deleting path=%r' % path)
return flag
[docs]def remove_existing_fpaths(fpath_list, verbose=VERBOSE, quiet=QUIET,
strict=False, print_caller=PRINT_CALLER, lbl='files'):
""" checks existance before removing. then tries to remove exisint paths """
import utool as ut
if print_caller:
print(util_dbg.get_caller_name(range(1, 4)) + ' called remove_existing_fpaths')
fpath_list_ = ut.filter_Nones(fpath_list)
exists_list = list(map(exists, fpath_list_))
if verbose:
nTotal = len(fpath_list)
nValid = len(fpath_list_)
nExist = sum(exists_list)
print('[util_path.remove_existing_fpaths] request delete of %d %s' % (
nTotal, lbl))
if nValid != nTotal:
print(('[util_path.remove_existing_fpaths] '
'trying to delete %d/%d non None %s ') %
(nValid, nTotal, lbl))
print(('[util_path.remove_existing_fpaths] '
' %d/%d exist and need to be deleted')
% (nExist, nValid))
existing_fpath_list = ut.compress(fpath_list_, exists_list)
return remove_fpaths(existing_fpath_list, verbose=verbose, quiet=quiet,
strict=strict, print_caller=False, lbl=lbl)
[docs]def remove_fpaths(fpath_list, verbose=VERBOSE, quiet=QUIET, strict=False,
print_caller=PRINT_CALLER, lbl='files'):
"""
Removes multiple file paths
"""
if print_caller:
print(util_dbg.get_caller_name(range(1, 4)) + ' called remove_fpaths')
nTotal = len(fpath_list)
_verbose = (not quiet and nTotal > 0) or VERYVERBOSE
if _verbose:
print('[util_path.remove_fpaths] try removing %d %s' % (nTotal, lbl))
nRemoved = 0
for fpath in fpath_list:
try:
os.remove(fpath) # Force refresh
nRemoved += 1
except OSError as ex:
if VERYVERBOSE:
print('WARNING: Could not remove fpath = %r' % (fpath,))
if strict:
util_dbg.printex(ex, 'Could not remove fpath = %r' % (fpath,),
iswarning=False)
raise
pass
if _verbose:
print('[util_path.remove_fpaths] ... removed %d / %d %s' % (
nRemoved, nTotal, lbl))
return nRemoved
remove_file_list = remove_fpaths # backwards compatible
[docs]def longest_existing_path(_path):
r"""
Returns the longest root of _path that exists
Args:
_path (str): path string
Returns:
str: _path - path string
CommandLine:
python -m utool.util_path --exec-longest_existing_path
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> target = dirname(ut.__file__)
>>> _path = join(target, 'nonexist/foobar')
>>> existing_path = longest_existing_path(_path)
>>> result = ('existing_path = %s' % (str(existing_path),))
>>> print(result)
>>> assert existing_path == target
"""
existing_path = _path
while True:
_path_new = os.path.dirname(existing_path)
if exists(_path_new):
existing_path = _path_new
break
if _path_new == existing_path:
print('!!! [utool] This is a very illformated path indeed.')
existing_path = ''
break
existing_path = _path_new
return existing_path
[docs]def get_path_type(path_):
r"""
returns if a path is a file, directory, link, or mount
"""
path_type = ''
if isfile(path_):
path_type += 'file'
if isdir(path_):
path_type += 'directory'
if islink(path_):
path_type += 'link'
if ismount(path_):
path_type += 'mount'
return path_type
[docs]def checkpath(path_, verbose=VERYVERBOSE, n=None, info=VERYVERBOSE):
r""" verbose wrapper around ``os.path.exists``
Returns:
true if ``path_`` exists on the filesystem show only the
top `n` directories
Args:
path_ (str): path string
verbose (bool): verbosity flag(default = False)
n (int): (default = None)
info (bool): (default = False)
CommandLine:
python -m utool.util_path --test-checkpath
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> path_ = ut.__file__
>>> verbose = True
>>> n = None
>>> info = False
>>> result = checkpath(path_, verbose, n, info)
>>> print(result)
True
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> path_ = ut.__file__ + 'foobar'
>>> verbose = True
>>> result = checkpath(path_, verbose, n=None, info=True)
>>> print(result)
False
"""
assert isinstance(path_, six.string_types), (
'path_=%r is not a string. type(path_) = %r' % (path_, type(path_)))
path_ = normpath(path_)
if sys.platform.startswith('win32'):
# convert back to windows style path if using unix style
if path_.startswith('\\'):
dirs = path_.split('\\')
if len(dirs) > 1 and len(dirs[0]) == 0 and len(dirs[1]) == 1:
dirs[1] = dirs[1].upper() + ':'
path_ = '\\'.join(dirs[1:])
does_exist = exists(path_)
if verbose:
#print_('[utool] checkpath(%r)' % (path_))
pretty_path = path_ndir_split(path_, n)
caller_name = util_dbg.get_caller_name(allow_genexpr=False)
print('[%s] checkpath(%r)' % (caller_name, pretty_path))
if does_exist:
path_type = get_path_type(path_)
#path_type = 'file' if isfile(path_) else 'directory'
print('[%s] ...(%s) exists' % (caller_name, path_type,))
else:
print('[%s] ... does not exist' % (caller_name))
if not does_exist and info:
#print('[util_path] ! Does not exist')
_longest_path = longest_existing_path(path_)
_longest_path_type = get_path_type(_longest_path)
print('[util_path] ... The longest existing path is: %r' % _longest_path)
print('[util_path] ... and has type %r' % (_longest_path_type,))
return does_exist
[docs]def ensurepath(path_, verbose=VERYVERBOSE):
""" DEPRICATE - alias - use ensuredir instead """
return ensuredir(path_, verbose=verbose)
[docs]def ensuredir(path_, verbose=VERYVERBOSE, info=False, mode=0o1777):
r"""
Ensures that directory will exist. creates new dir with sticky bits by
default
"""
if not checkpath(path_, verbose=verbose, info=info):
if verbose:
print('[util_path] mkdir(%r)' % path_)
#os.makedirs(path_)
try:
os.makedirs(normpath(path_), mode=mode)
except OSError as ex:
util_dbg.printex(ex,
'check that the longest existing path '
'is not a bad windows symlink.')
raise
return path_
#return True
# ---File Copy---
[docs]def copy_worker(args):
""" for util_parallel.generate """
src, dst = args
try:
shutil.copy2(src, dst)
except OSError:
return False
except shutil.Error:
pass
return True
[docs]def copy_files_to(src_fpath_list, dst_dpath=None, dst_fpath_list=None,
overwrite=False, verbose=True, veryverbose=False):
"""
parallel copier
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import *
>>> import utool as ut
>>> overwrite = False
>>> veryverbose = False
>>> verbose = True
>>> src_fpath_list = [ut.grab_test_imgpath(key)
>>> for key in ut.get_valid_test_imgkeys()]
>>> dst_dpath = ut.get_app_resource_dir('utool', 'filecopy_tests')
>>> copy_files_to(src_fpath_list, dst_dpath, overwrite=overwrite,
>>> verbose=verbose)
"""
from utool import util_list
from utool import util_parallel
if verbose:
print('[util_path] +--- COPYING FILES ---')
print('[util_path] * len(src_fpath_list) = %r' % (len(src_fpath_list)))
print('[util_path] * dst_dpath = %r' % (dst_dpath,))
if dst_fpath_list is None:
ensuredir(dst_dpath, verbose=veryverbose)
dst_fpath_list = [join(dst_dpath, basename(fpath))
for fpath in src_fpath_list]
else:
assert dst_dpath is None, 'dst_dpath was specified but overrided'
assert len(dst_fpath_list) == len(src_fpath_list), 'bad correspondence'
exists_list = list(map(exists, dst_fpath_list))
if verbose:
print('[util_path] * %d files already exist dst_dpath' % (
sum(exists_list),))
if not overwrite:
notexists_list = util_list.not_list(exists_list)
dst_fpath_list_ = util_list.compress(dst_fpath_list, notexists_list)
src_fpath_list_ = util_list.compress(src_fpath_list, notexists_list)
else:
dst_fpath_list_ = dst_fpath_list
src_fpath_list_ = src_fpath_list
args_list = zip(src_fpath_list_, dst_fpath_list_)
_gen = util_parallel.generate(copy_worker, args_list,
nTasks=len(src_fpath_list_))
success_list = list(_gen)
#success_list = copy_list(src_fpath_list_, dst_fpath_list_)
if verbose:
print('[util_path] * Copied %d / %d' % (sum(success_list),
len(src_fpath_list)))
print('[util_path] L___ DONE COPYING FILES ___')
[docs]def copy(src, dst, overwrite=True, deeplink=True, verbose=True, dryrun=False):
import utool as ut
if ut.isiterable(src):
if not ut.isiterable(dst):
# list to non list
ut.copy_files_to(src, dst, overwrite=overwrite, verbose=verbose)
else:
# list to list
ut.copy_files_to(src, dst_fpath_list=dst, overwrite=overwrite,
verbose=verbose)
else:
return copy_single(src, dst, overwrite=overwrite, deeplink=deeplink,
dryrun=dryrun, verbose=verbose)
[docs]def copy_single(src, dst, overwrite=True, verbose=True, deeplink=True,
dryrun=False):
r"""
Args:
src (str): file or directory to copy
dst (str): directory or new file to copy to
Copies src file or folder to dst.
If src is a folder this copy is recursive.
"""
try:
if exists(src):
if not isdir(src) and isdir(dst):
# copying file to directory
dst = join(dst, basename(src))
if exists(dst):
if overwrite:
prefix = 'C+O'
if verbose:
print('[util_path] [Copying + Overwrite]:')
else:
prefix = 'Skip'
if verbose:
print('[%s] ->%s' % (prefix, dst))
return
else:
prefix = 'C'
if verbose:
if dryrun:
print('[util_path] [DryRun]: ')
else:
print('[util_path] [Copying]: ')
if verbose:
print('[%s] | %s' % (prefix, src))
print('[%s] ->%s' % (prefix, dst))
if not dryrun:
if not deeplink and islink(src):
linkto = os.readlink(src)
symlink(linkto, dst)
elif isdir(src):
print('isdir')
shutil.copytree(src, dst)
else:
shutil.copy2(src, dst)
else:
prefix = 'Miss'
if verbose:
print('[util_path] [Cannot Copy]: ')
print('[%s] src=%s does not exist!' % (prefix, src))
print('[%s] dst=%s' % (prefix, dst))
except Exception as ex:
from utool import util_dbg
util_dbg.printex(ex, 'Error copying single', keys=['src', 'dst'])
raise
[docs]def copy_all(src_dir, dest_dir, glob_str_list, recursive=False):
ensuredir(dest_dir)
if not isinstance(glob_str_list, list):
glob_str_list = [glob_str_list]
for root, dirs, files in os.walk(src_dir):
for dname_ in dirs:
for glob_str in glob_str_list:
if fnmatch.fnmatch(dname_, glob_str):
src = normpath(join(src_dir, dname_))
dst = normpath(join(dest_dir, dname_))
ensuredir(dst)
for fname_ in files:
for glob_str in glob_str_list:
if fnmatch.fnmatch(fname_, glob_str):
src = normpath(join(src_dir, fname_))
dst = normpath(join(dest_dir, fname_))
copy(src, dst)
if not recursive:
break
[docs]def copy_list(src_list, dst_list, lbl='Copying',
ioerr_ok=False, sherro_ok=False, oserror_ok=False):
""" Copies all data and stat info """
# Feb - 6 - 2014 Copy function
task_iter = zip(src_list, dst_list)
def docopy(src, dst):
try:
shutil.copy2(src, dst)
except OSError:
if ioerr_ok:
return False
raise
except shutil.Error:
if sherro_ok:
return False
raise
except IOError:
if ioerr_ok:
return False
raise
return True
progiter = util_progress.ProgIter(task_iter, adjust=True, lbl=lbl)
success_list = [docopy(src, dst) for (src, dst) in progiter]
return success_list
[docs]def move(src, dst, lbl='Moving'):
return move_list([src], [dst], lbl)
[docs]def move_list(src_list, dst_list, lbl='Moving'):
# Feb - 6 - 2014 Move function
def trymove(src, dst):
try:
shutil.move(src, dst)
except OSError:
return False
return True
task_iter = zip(src_list, dst_list)
progiter = util_progress.ProgIter(task_iter, lbl=lbl, adjust=True)
success_list = [trymove(src, dst) for (src, dst) in progiter]
return success_list
[docs]def win_shortcut(source, link_name):
"""
Attempt to create windows shortcut
TODO: TEST / FIXME
References:
http://stackoverflow.com/questions/1447575/symlinks-on-windows
"""
if True:
import ctypes
kdll = ctypes.windll.LoadLibrary("kernel32.dll")
code = 1 if isdir(source) else 0
kdll.CreateSymbolicLinkA(source, link_name, code)
else:
import ctypes
csl = ctypes.windll.kernel32.CreateSymbolicLinkW
csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
csl.restype = ctypes.c_ubyte
flags = 1 if isdir(source) else 0
retval = csl(link_name, source, flags)
if retval == 0:
#warn_msg = '[util_path] Unable to create symbolic link on windows.'
#print(warn_msg)
#warnings.warn(warn_msg, category=UserWarning)
if checkpath(link_name):
return True
raise ctypes.WinError()
[docs]def symlink(path, link, noraise=False):
"""
Attempt to create unix or windows symlink
TODO: TEST / FIXME
Args:
path (str): path to real file or directory
link (str): path to desired location for symlink
noraise (bool):
"""
path = normpath(path)
link = normpath(link)
if os.path.islink(link):
print('[util_path] symlink %r exists' % (link))
return
print('[util_path] Creating symlink: path=%r link_name=%r' % (path, link))
try:
os_symlink = getattr(os, "symlink", None)
if callable(os_symlink):
os_symlink(path, link)
else:
win_shortcut(path, link)
except Exception as ex:
checkpath(link, True)
checkpath(path, True)
import utool as ut
ut.printex(ex, 'error making symlink', iswarning=noraise)
if not noraise:
raise
[docs]def file_bytes(fpath):
r"""
Args:
fpath (str): file path string
Returns:
int: size of file in bytes
"""
return os.stat(fpath).st_size
[docs]def file_megabytes(fpath):
r"""
Args:
fpath (str): file path string
Returns:
float: size of file in megabytes
"""
return os.stat(fpath).st_size / (2.0 ** 20)
[docs]def glob_python_modules(dirname, **kwargs):
return glob(dirname, '*.py', recursive=True, with_dirs=False)
[docs]def glob(dpath, pattern=None, recursive=False, with_files=True, with_dirs=True,
maxdepth=None, exclude_dirs=[], fullpath=True, **kwargs):
r"""
Globs directory for pattern
Args:
dpath (str): directory path or pattern
pattern (str or list): pattern or list of patterns
(use only if pattern is not in dpath)
recursive (bool): (default = False)
with_files (bool): (default = True)
with_dirs (bool): (default = True)
maxdepth (None): (default = None)
exclude_dirs (list): (default = [])
Returns:
list: path_list
SeeAlso:
iglob
CommandLine:
python -m utool.util_path --test-glob
python -m utool.util_path --exec-glob:1
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> from os.path import dirname
>>> dpath = dirname(ut.__file__)
>>> pattern = '__*.py'
>>> recursive = True
>>> with_files = True
>>> with_dirs = True
>>> maxdepth = None
>>> fullpath = False
>>> exclude_dirs = ['_internal', join(dpath, 'experimental')]
>>> print('exclude_dirs = ' + ut.list_str(exclude_dirs))
>>> path_list = glob(dpath, pattern, recursive, with_files, with_dirs,
>>> maxdepth, exclude_dirs, fullpath)
>>> result = ('path_list = %s' % (ut.list_str(path_list),))
>>> result = result.replace(r'\\', '/')
>>> print(result)
path_list = [
'__init__.py',
'__main__.py',
'tests/__init__.py',
]
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> dpath = dirname(ut.__file__) + '/__*.py'
>>> path_list = glob(dpath)
>>> result = ('path_list = %s' % (str(path_list),))
>>> print(result)
"""
gen = iglob(dpath, pattern, recursive=recursive,
with_files=with_files, with_dirs=with_dirs, maxdepth=maxdepth,
fullpath=fullpath, exclude_dirs=exclude_dirs, **kwargs)
path_list = list(gen)
return path_list
[docs]def iglob(dpath, pattern=None, recursive=False, with_files=True, with_dirs=True,
maxdepth=None, exclude_dirs=[], fullpath=True, **kwargs):
r"""
Iteratively globs directory for pattern
FIXME:
This function has a speed issue
Args:
dpath (str): directory path
pattern (str):
recursive (bool): (default = False)
with_files (bool): (default = True)
with_dirs (bool): (default = True)
maxdepth (None): (default = None)
exclude_dirs (list): (default = [])
Yields:
path
References:
http://stackoverflow.com/questions/19859840/excluding-dirs-in-os-walk
"""
from utool import util_iter
if kwargs.get('verbose', False): # log what i'm going to do
print('[util_path] glob(dpath=%r)' % truepath(dpath,))
debug = False
if pattern is None:
# separate extract pattern from dpath
if debug:
print('[iglob] parsing dpath = %r' % (dpath,))
dpath_ = dpath
dpath = longest_existing_path(dpath_)
pattern = relpath(dpath_, dpath)
else:
# hack check for pattern
GLOB_PATS = ['*', '?']
for _ in GLOB_PATS:
assert dpath.find(_) == -1, (
'warning: pattern _=%r in dpath, but a pattern was specified' %
(_,))
if isinstance(pattern, list):
# overload pattern with list
pattern_list = pattern
subiters = (
iglob(dpath, pattern=pattern, recursive=recursive,
with_files=with_files, with_dirs=with_dirs,
maxdepth=maxdepth, exclude_dirs=exclude_dirs,
fullpath=fullpath, **kwargs)
for pattern in pattern_list
)
for item in util_iter.iflatten(subiters):
yield item
raise StopIteration
if debug:
print('[iglob] pattern = %r' % (pattern,))
print('[iglob] dpath = %r' % (dpath,))
nFiles = 0
nDirs = 0
current_depth = 0
dpath_ = truepath(dpath)
posx1 = len(dpath_) + len(os.path.sep)
#exclude_dirs_rel = [relpath(dpath_, dir_) for dir_ in exclude_dirs]
#exclude_dirs_rel = [relpath(dpath_, dir_) for dir_ in exclude_dirs]
#print('\n\n\n')
#import utool as ut
#print('exclude_dirs = %s' % (ut.list_str(exclude_dirs),))
for root, dirs, files in os.walk(dpath_, topdown=True):
# Modifying dirs in-place will prune the subsequent files and
# directories visitied by os.walk
# References:
# http://stackoverflow.com/questions/19859840/excluding-directories-in-os-walk
rel_root = relpath(root, dpath_)
rel_root2 = relpath(root, dirname(dpath_))
#print('rel_root = %r' % (rel_root,))
#if len(dirs) > 0:
# print('dirs = %s' % (ut.list_str([join(rel_root, d) for d in dirs]),))
if len(exclude_dirs) > 0:
dirs[:] = [d for d in dirs if normpath(join(rel_root, d)) not in exclude_dirs]
# hack
dirs[:] = [d for d in dirs if normpath(join(rel_root2, d)) not in exclude_dirs]
# check abs path as well
dirs[:] = [d for d in dirs if normpath(join(root, d)) not in exclude_dirs]
# yeild data
# print it only if you want
if maxdepth is not None:
current_depth = root[posx1:].count(os.path.sep)
if maxdepth <= current_depth:
continue
#print('-----------')
#print(current_depth)
#print(root)
#print('==')
#print(dirs)
#print('-----------')
if with_files:
for fname in fnmatch.filter(files, pattern):
nFiles += 1
fpath = join(root, fname)
if fullpath:
yield fpath
else:
yield relpath(fpath, dpath_)
if with_dirs:
for dname in fnmatch.filter(dirs, pattern):
dpath = join(root, dname)
nDirs += 1
if fullpath:
yield dpath
else:
yield relpath(dpath, dpath_)
if not recursive:
break
if kwargs.get('verbose', False): # log what i've done
nTotal = nDirs + nFiles
print('[util_path] iglob Found: %d' % (nTotal))
# --- Images ----
[docs]def num_images_in_dir(path):
"""
returns the number of images in a directory
"""
num_imgs = 0
for root, dirs, files in os.walk(path):
for fname in files:
if matches_image(fname):
num_imgs += 1
return num_imgs
[docs]def matches_image(fname):
""" returns true if a filename matches an image pattern """
fname_ = fname.lower()
img_pats = ['*' + ext for ext in IMG_EXTENSIONS]
return any([fnmatch.fnmatch(fname_, pat) for pat in img_pats])
[docs]def dirsplit(path):
r"""
Args:
path (str):
Returns:
list: compoments of the path
CommandLine:
python -m utool.util_path --exec-dirsplit
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> paths = []
>>> paths.append('E:/window file/foo')
>>> paths.append('/normal/foo')
>>> paths.append('~/relative/path')
>>> results = [dirsplit(path) for path in paths]
>>> import re
>>> results2 = [re.split('\\/', path) for path in paths]
>>> print(results2)
>>> result = ut.repr2(results)
>>> print(result)
"""
#return path.split(os.sep)
parts = []
remain = path
part = True
#while True:
while part != '' and remain != '':
remain, part = split(remain)
parts.append(part)
parts = [p for p in parts if p != '']
if remain != '':
parts.append(remain)
parts = parts[::-1]
return parts
[docs]def fpaths_to_fnames(fpath_list):
"""
Args:
fpath_list (list of strs): list of file-paths
Returns:
fname_list (list of strs): list of file-names
"""
fname_list = [split(fpath)[1] for fpath in fpath_list]
return fname_list
[docs]def fnames_to_fpaths(fname_list, path):
fpath_list = [join(path, fname) for fname in fname_list]
return fpath_list
[docs]def get_modpath_from_modname(modname, prefer_pkg=False, prefer_main=False):
r"""
Args:
modname (str):
Returns:
str: module_dir
CommandLine:
python -m utool.util_path --test-get_modpath_from_modname
Setup:
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> utool_dir = dirname(dirname(ut.__file__))
Example:
>>> # ENABLE_DOCTEST
>>> modname = 'utool.util_path'
>>> module_dir = get_modpath_from_modname(modname)
>>> result = ut.truepath_relative(module_dir, utool_dir)
>>> result = ut.ensure_unixslash(result)
>>> print(result)
utool/util_path.py
Example:
>>> # ENABLE_DOCTEST
>>> modname = 'utool._internal'
>>> module_dir = get_modpath_from_modname(modname, prefer_pkg=True)
>>> result = ut.truepath_relative(module_dir, utool_dir)
>>> result = ut.ensure_unixslash(result)
>>> print(result)
utool/_internal
Example:
>>> # ENABLE_DOCTEST
>>> modname = 'utool'
>>> module_dir = get_modpath_from_modname(modname)
>>> result = ut.truepath_relative(module_dir, utool_dir)
>>> result = ut.ensure_unixslash(result)
>>> print(result)
utool/__init__.py
"""
import importlib
if isinstance(modname, six.string_types):
module = importlib.import_module(modname)
else:
module = modname # Hack
modpath = module.__file__.replace('.pyc', '.py')
initname = '__init__.py'
mainname = '__main__.py'
if prefer_pkg:
if modpath.endswith(initname) or modpath.endswith(mainname):
modpath = modpath[:-len(initname)]
if prefer_main:
if modpath.endswith(initname):
main_modpath = modpath[:-len(initname)] + mainname
if exists(main_modpath):
modpath = main_modpath
#modname = modname.replace('.__init__', '').strip()
#module_dir = get_module_dir(module)
return modpath
[docs]def get_module_dir(module, *args):
module_dir = truepath(dirname(module.__file__))
if len(args) > 0:
module_dir = join(module_dir, *args)
return module_dir
[docs]def ensure_unixslash(path):
return path.replace('\\', '/')
[docs]def ensure_crossplat_path(path, winroot='C:'):
r"""
ensure_crossplat_path
Args:
path (str):
Returns:
str: crossplat_path
Example(DOCTEST):
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> path = r'C:\somedir'
>>> cplat_path = ensure_crossplat_path(path)
>>> result = cplat_path
>>> print(result)
C:/somedir
"""
cplat_path = path.replace('\\', '/')
if cplat_path == winroot:
cplat_path += '/'
return cplat_path
[docs]def ensure_native_path(path, winroot='C:'):
import utool as ut
if ut.WIN32 and path.startswith('/') or path.startswith('\\'):
path = winroot + path
[docs]def get_relative_modpath(module_fpath):
"""
Returns path to module relative to the package root
Args:
module_fpath (str): module filepath
Returns:
str: modname
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> module_fpath = ut.util_path.__file__
>>> rel_modpath = ut.get_relative_modpath(module_fpath)
>>> rel_modpath = rel_modpath.replace('.pyc', '.py') # allow pyc or py
>>> result = ensure_crossplat_path(rel_modpath)
>>> print(result)
utool/util_path.py
"""
modsubdir_list = get_module_subdir_list(module_fpath)
_, ext = splitext(module_fpath)
rel_modpath = join(*modsubdir_list) + ext
rel_modpath = ensure_crossplat_path(rel_modpath)
return rel_modpath
[docs]def get_modname_from_modpath(module_fpath):
"""
returns importable name from file path
get_modname_from_modpath
Args:
module_fpath (str): module filepath
Returns:
str: modname
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> module_fpath = ut.util_path.__file__
>>> modname = ut.get_modname_from_modpath(module_fpath)
>>> result = modname
>>> print(result)
utool.util_path
"""
modsubdir_list = get_module_subdir_list(module_fpath)
modname = '.'.join(modsubdir_list)
modname = modname.replace('.__init__', '').strip()
modname = modname.replace('.__main__', '').strip()
return modname
[docs]def get_module_subdir_list(module_fpath):
"""
get_module_subdir_list
Args:
module_fpath (str):
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> module_fpath = ut.util_path.__file__
>>> modsubdir_list = get_module_subdir_list(module_fpath)
>>> result = modsubdir_list
>>> print(result)
['utool', 'util_path']
"""
module_fpath = truepath(module_fpath)
dpath, fname_ext = split(module_fpath)
fname, ext = splitext(fname_ext)
full_dpath = dpath
dpath = full_dpath
_modsubdir_list = [fname]
while is_module_dir(dpath):
dpath, dname = split(dpath)
_modsubdir_list.append(dname)
modsubdir_list = _modsubdir_list[::-1]
return modsubdir_list
[docs]def ls(path, pattern='*'):
""" like unix ls - lists all files and dirs in path"""
path_iter = glob(path, pattern, recursive=False)
return sorted(list(path_iter))
[docs]def ls_dirs(path, pattern='*'):
dir_iter = list(glob(path, pattern, recursive=False, with_files=False))
return sorted(list(dir_iter))
[docs]def ls_modulefiles(path, private=True, full=True, noext=False):
module_file_list = ls(path, '*.py')
module_file_iter = iter(module_file_list)
if not private:
module_file_iter = filterfalse(is_private_module, module_file_iter)
if not full:
module_file_iter = map(basename, module_file_iter)
if noext:
module_file_iter = (splitext(path)[0] for path in module_file_iter)
return list(module_file_iter)
[docs]def ls_moduledirs(path, private=True, full=True):
""" lists all dirs which are python modules in path """
dir_list = ls_dirs(path)
module_dir_iter = filter(is_module_dir, dir_list)
if not private:
module_dir_iter = filterfalse(is_private_module, module_dir_iter)
if not full:
module_dir_iter = map(basename, module_dir_iter)
return list(module_dir_iter)
[docs]def get_basename_noext_list(path_list):
return [basename_noext(path) for path in path_list]
[docs]def get_ext_list(path_list):
return [splitext(path)[1] for path in path_list]
[docs]def get_basepath_list(path_list):
return [split(path)[0] for path in path_list]
[docs]def basename_noext(path):
return splitext(basename(path))[0]
[docs]def append_suffixlist_to_namelist(name_list, suffix_list):
""" adds a suffix to the path before the extension
if name_list is a path_list the basepath is stripped away """
assert len(name_list) == len(suffix_list)
#basepath_list = utool.get_basepath_list(name_list)
gnamenoext_list = get_basename_noext_list(name_list)
ext_list = get_ext_list(name_list)
new_name_list = [name + suffix + ext for name, suffix, ext in
zip(gnamenoext_list, suffix_list, ext_list)]
return new_name_list
[docs]def is_private_module(path):
return basename(path).startswith('__')
[docs]def is_python_module(path):
return path.endswith('.py') or path.endswith('.pyc')
[docs]def is_module_dir(path):
"""
Args:
path (str)
Returns:
flag: True if path contains an __init__ file
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> path = truepath('~/code/utool/utool')
>>> flag = is_module_dir(path)
>>> result = (flag)
>>> print(result)
"""
return exists(join(path, '__init__.py'))
[docs]def list_images(img_dpath_, ignore_list=[], recursive=False, fullpath=False,
full=None, sort=True):
r"""
Returns a list of images in a directory. By default returns relative paths.
TODO: rename to ls_images
TODO: Change all instances of fullpath to full
Args:
img_dpath_ (str):
ignore_list (list): (default = [])
recursive (bool): (default = False)
fullpath (bool): (default = False)
full (None): (default = None)
sort (bool): (default = True)
Returns:
list: gname_list
CommandLine:
python -m utool.util_path --exec-list_images
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> img_dpath_ = '?'
>>> ignore_list = []
>>> recursive = False
>>> fullpath = False
>>> full = None
>>> sort = True
>>> gname_list = list_images(img_dpath_, ignore_list, recursive,
>>> fullpath, full, sort)
>>> result = ('gname_list = %s' % (str(gname_list),))
>>> print(result)
"""
#if not QUIET:
# print(ignore_list)
if full is not None:
fullpath = fullpath or full
img_dpath = realpath(img_dpath_)
ignore_set = set(ignore_list)
gname_list_ = []
assertpath(img_dpath)
# Get all the files in a directory recursively
true_imgpath = truepath(img_dpath)
for root, dlist, flist in os.walk(true_imgpath):
rel_dpath = relpath(root, img_dpath)
# Ignore directories
if any([dname in ignore_set for dname in dirsplit(rel_dpath)]):
continue
for fname in iter(flist):
gname = join(rel_dpath, fname).replace('\\', '/')
if gname.startswith('./'):
gname = gname[2:]
if matches_image(gname):
# Ignore Files
if gname in ignore_set:
continue
if fullpath:
gpath = join(img_dpath, gname)
gname_list_.append(gpath)
else:
gname_list_.append(gname)
if not recursive:
break
# Filter out non images or ignorables
#gname_list = [gname_ for gname_ in iter(gname_list_)
# if gname_ not in ignore_set and matches_image(gname_)]
if sort:
gname_list = sorted(gname_list_)
return gname_list
ls_images = list_images
[docs]def assertpath(path_, msg='', **kwargs):
""" Asserts that a patha exists """
if NO_ASSERTS:
return
if path_ is None:
raise AssertionError('path is None! %s' % (path_, msg))
if path_ == '':
raise AssertionError('path=%r is the empty string! %s' % (path_, msg))
if not checkpath(path_, **kwargs):
raise AssertionError('path=%r does not exist! %s' % (path_, msg))
assert_exists = assertpath
#def assert_exists(path, msg='', **kwargs):
# if NO_ASSERTS:
# return
# return assertpath(path, msg, **kwargs)
# #assert exists(path), 'path=%r does not exist! %s' % (path, msg)
[docs]def pathsplit_full(path):
""" splits all directories in path into a list """
return path.replace('\\', '/').split('/')
[docs]def get_standard_exclude_dnames():
return ['lib.linux-x86_64-2.7', 'dist', 'build', '_page', '_doc',
'utool.egg-info', '.git']
[docs]def get_standard_include_patterns():
return ['*.py', '*.cxx', '*.cpp', '*.hxx', '*.hpp', '*.c', '*.h', '*.vim']
[docs]def matching_fnames(dpath_list, include_patterns, exclude_dirs=[],
greater_exclude_dirs=[], recursive=True):
r"""
matching_fnames. walks dpath lists returning all directories that match the
requested pattern.
Args:
dpath_list (list):
include_patterns (str):
exclude_dirs (None):
recursive (bool):
References:
# TODO: fix names and behavior of exclude_dirs and greater_exclude_dirs
http://stackoverflow.com/questions/19859840/excluding-directories-in-os-walk
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> dpath_list = [dirname(dirname(ut.__file__))]
>>> include_patterns = get_standard_include_patterns()
>>> exclude_dirs = ['_page']
>>> greater_exclude_dirs = get_standard_exclude_dnames()
>>> recursive = True
>>> fpath_gen = matching_fnames(dpath_list, include_patterns, exclude_dirs,
>>> greater_exclude_dirs, recursive)
>>> result = list(fpath_gen)
>>> print('\n'.join(result))
"""
if isinstance(dpath_list, six.string_types):
dpath_list = [dpath_list]
for dpath in dpath_list:
for root, dname_list, fname_list in os.walk(dpath):
# Look at all subdirs
subdirs = pathsplit_full(relpath(root, dpath))
# HACK:
if any([dir_ in greater_exclude_dirs for dir_ in subdirs]):
continue
# Look at one subdir
if basename(root) in exclude_dirs:
continue
for name in fname_list:
# For the filesnames which match the patterns
if any([fnmatch.fnmatch(name, pat) for pat in include_patterns]):
yield join(root, name)
#fname_list.append((root, name))
if not recursive:
break
#return fname_list
[docs]def sed(regexpr, repl, force=False, recursive=False, dpath_list=None,
fpath_list=None, verbose=None):
"""
Python implementation of sed. NOT FINISHED
searches and replaces text in files
Args:
regexpr (str): regx patterns to find
repl (str): text to replace
force (bool):
recursive (bool):
dpath_list (list): directories to search (defaults to cwd)
"""
#_grep(r, [repl], dpath_list=dpath_list, recursive=recursive)
include_patterns = ['*.py', '*.cxx', '*.cpp', '*.hxx', '*.hpp', '*.c', '*.h', '*.html']
if dpath_list is None:
dpath_list = [os.getcwd()]
if verbose is None:
verbose = ut.NOT_QUIET
if fpath_list is None:
greater_exclude_dirs = get_standard_exclude_dnames()
exclude_dirs = []
fpath_generator = matching_fnames(
dpath_list, include_patterns, exclude_dirs, greater_exclude_dirs,
recursive=recursive)
#fpath_generator = matching_fnames(dpath_list, include_patterns, recursive=recursive)
else:
fpath_generator = fpath_list
if verbose:
print('sed-ing %r' % (dpath_list,))
print(' * regular expression : %r' % (regexpr,))
print(' * replacement : %r' % (repl,))
print(' * recursive: %r' % (recursive,))
print(' * force: %r' % (force,))
from utool import util_str
print(' * fpath_list: %s' % (util_str.repr3(fpath_list),))
regexpr = extend_regex(regexpr)
#if '\x08' in regexpr:
# print('Remember \\x08 != \\b')
# print('subsituting for you for you')
# regexpr = regexpr.replace('\x08', '\\b')
# print(' * regular expression : %r' % (regexpr,))
# Walk through each directory recursively
num_changed = 0
fpaths_changed = []
for fpath in fpath_generator:
changed_lines = sedfile(fpath, regexpr, repl, force, verbose=verbose)
if changed_lines is not None:
fpaths_changed.append(fpath)
num_changed += len(changed_lines)
import utool as ut
print('fpaths_changed = %s' % (ut.repr3(sorted(fpaths_changed)),))
print('total lines changed = %r' % (num_changed,))
[docs]def sedfile(fpath, regexpr, repl, force=False, verbose=True, veryverbose=False):
"""
Executes sed on a specific file
Args:
fpath (str): file path string
regexpr (str):
repl (str):
force (bool): (default = False)
verbose (bool): verbosity flag(default = True)
veryverbose (bool): (default = False)
Returns:
list: changed_lines
CommandLine:
python -m utool.util_path --exec-sedfile --show
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> fpath = ut.get_modpath_from_modname(ut.util_path)
>>> regexpr = 'sedfile'
>>> repl = 'saidfile'
>>> force = False
>>> verbose = True
>>> veryverbose = False
>>> changed_lines = sedfile(fpath, regexpr, repl, force, verbose, veryverbose)
>>> result = ('changed_lines = %s' % (ut.repr3(changed_lines),))
>>> print(result)
"""
# TODO: move to util_edit
path, name = split(fpath)
new_file_lines = []
if veryverbose:
print('[sedfile] fpath=%r' % fpath)
print('[sedfile] regexpr=%r' % regexpr)
print('[sedfile] repl=%r' % repl)
print('[sedfile] force=%r' % force)
with open(fpath, 'r') as file:
file_lines = file.readlines()
# Search each line for the desired regexpr
new_file_lines = [re.sub(regexpr, repl, line) for line in file_lines]
changed_lines = [(newline, line)
for newline, line in zip(new_file_lines, file_lines)
if newline != line]
nChanged = len(changed_lines)
if nChanged > 0:
rel_fpath = relpath(fpath, os.getcwd())
print(' * %s changed %d lines in %r ' %
(['(dry-run)', '(real-run)'][force], nChanged, rel_fpath))
print(' * --------------------')
import utool as ut
new_file_lines = ut.lmap(ut.ensure_unicode, new_file_lines)
new_file = ''.join(new_file_lines)
#print(new_file.replace('\n','\n))
if verbose:
if True:
import utool as ut
old_file = ut.ensure_unicode(
''.join(ut.lmap(ut.ensure_unicode, file_lines)))
ut.print_difftext(old_file, new_file)
else:
changed_new, changed_old = zip(*changed_lines)
prefixold = ' * old (%d, %r): \n | ' % (nChanged, name)
prefixnew = ' * new (%d, %r): \n | ' % (nChanged, name)
print(prefixold + (' | '.join(changed_old)).strip('\n'))
print(' * ____________________')
print(prefixnew + (' | '.join(changed_new)).strip('\n'))
print(' * --------------------')
print(' * =====================================================')
# Write back to file
if force:
print(' ! WRITING CHANGES')
with open(fpath, 'w') as file:
file.write(new_file.encode('utf8'))
else:
print(' dry run')
return changed_lines
#elif verbose:
# print('Nothing changed')
return None
@profile
[docs]def grepfile(fpath, regexpr_list, reflags=0, cache=None):
"""
grepfile - greps a specific file
Args:
fpath (str):
regexpr_list (list or str): pattern or list of patterns
Returns:
tuple (list, list): list of lines and list of line numbers
CommandLine:
python -m utool.util_path --exec-grepfile
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> fpath = ut.get_modpath_from_modname(ut.util_path)
>>> regexpr_list = ['foundthisline', '__future__']
>>> cache = None
>>> reflags = 0
>>> found_lines, found_lxs = ut.grepfile(fpath, regexpr_list)
>>> result = ut.repr3({'found_lines': found_lines, 'found_lxs': found_lxs})
>>> print(result)
>>> assert 7 in found_lxs
>>> others = ut.take_complement(found_lxs, [found_lxs.index(7)])
>>> assert others[0] == others[1]
"""
found_lines = []
found_lxs = []
# Ensure a list
islist = isinstance(regexpr_list, (list, tuple))
islist2 = isinstance(reflags, (list, tuple))
regexpr_list_ = regexpr_list if islist else [regexpr_list]
reflags_list = reflags if islist2 else [reflags] * len(regexpr_list_)
re_list = [re.compile(pat, flags=_flags)
for pat, _flags in zip(regexpr_list_, reflags_list)]
#print('regexpr_list_ = %r' % (regexpr_list_,))
#print('re_list = %r' % (re_list,))
import numpy as np
# Open file and search lines or use cache
if cache is None or fpath not in cache:
#with open(fpath, 'r') as file_:
# lines = list(file_.readlines())
from utool import util_io
lines = util_io.read_from(fpath, aslines=True, verbose=False)
#import utool as ut
#cumsum = ut.cumsum(map(len, lines))
cumsum = np.cumsum(list(map(len, lines)))
text = ''.join(lines)
if cache is not None:
cache[fpath] = (cumsum, text, lines)
else:
(cumsum, text, lines) = cache[fpath]
# Search each line for each pattern
old_method = False
if old_method:
for lx, line in enumerate(lines):
#for regexpr_ in regexpr_list_:
match_objects = [re_.search(line) for re_ in re_list]
for match in match_objects:
if match is not None:
found_lines.append(line)
found_lxs.append(lx)
else:
for re_ in re_list:
# FIXME: multiline mode doesnt work
for match_object in re_.finditer(text):
#print('match_object = %r' % (match_object,))
lxs = np.where(match_object.start() < cumsum)[0][0:1]
if len(lxs) == 1:
lx = lxs[0]
if lx > 0:
line_start = cumsum[lx - 1]
else:
line_start = 0
line_end = cumsum[lx]
line = text[line_start:line_end]
found_lines.append(line)
found_lxs.append(lx)
#[match_object.start > x for x in cumsum]
return found_lines, found_lxs
[docs]def testgrep():
"""
utprof.py -m utool.util_path --exec-testgrep
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> #dpath_list = [ut.truepath('~/code/utool/utool')]
>>> dpath_list = [ut.truepath(dirname(ut.__file__))]
>>> include_patterns = ['*.py']
>>> exclude_dirs = []
>>> regex_list = ['grepfile']
>>> verbose = True
>>> recursive = True
>>> result = ut.grep(regex_list, recursive, dpath_list, include_patterns,
>>> exclude_dirs)
>>> (found_fpath_list, found_lines_list, found_lxs_list) = result
>>> assert 'util_path.py' in list(map(basename, found_fpath_list))
"""
pass
# FIXME: util_test can't find the function if profile is enabled
@profile
[docs]def grep(regex_list, recursive=True, dpath_list=None, include_patterns=None,
exclude_dirs=[], greater_exclude_dirs=None, inverse=False,
verbose=VERBOSE, fpath_list=None, reflags=0, cache=None):
r"""
greps for patterns
Python implementation of grep. NOT FINISHED
Args:
regex_list (str or list): one or more patterns to find
recursive (bool):
dpath_list (list): directories to search (defaults to cwd)
include_patterns (list) : defaults to standard file extensions
Returns:
(list, list, list): (found_fpaths, found_lines_list, found_lxs_list)
CommandLine:
python -m utool.util_path --test-grep
utprof.py -m utool.util_path --exec-grep
utprof.py utool/util_path.py --exec-grep
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut
>>> #dpath_list = [ut.truepath('~/code/utool/utool')]
>>> dpath_list = [ut.truepath(dirname(ut.__file__))]
>>> include_patterns = ['*.py']
>>> exclude_dirs = []
>>> regex_list = ['grepfile']
>>> verbose = True
>>> recursive = True
>>> result = ut.grep(regex_list, recursive, dpath_list, include_patterns,
>>> exclude_dirs)
>>> (found_fpath_list, found_lines_list, found_lxs_list) = result
>>> assert 'util_path.py' in list(map(basename, found_fpath_list))
"""
from utool import util_regex
# from utool import util_str
from utool import util_list
if include_patterns is None:
include_patterns = get_standard_include_patterns()
if greater_exclude_dirs is None:
greater_exclude_dirs = get_standard_exclude_dnames()
# ensure list input
if isinstance(include_patterns, six.string_types):
include_patterns = [include_patterns]
if dpath_list is None:
dpath_list = [os.getcwd()]
if verbose:
recursive_stat_str = ['flat', 'recursive'][recursive]
print('[util_path] Greping (%s) %r for %r' % (recursive_stat_str,
dpath_list, regex_list))
print('[util_path] regex_list = %s' % (regex_list))
if isinstance(regex_list, six.string_types):
regex_list = [regex_list]
found_fpath_list = []
found_lines_list = []
found_lxs_list = []
# Walk through each directory recursively
if fpath_list is None:
fpath_generator = matching_fnames(dpath_list, include_patterns,
exclude_dirs, greater_exclude_dirs,
recursive=recursive)
else:
fpath_generator = fpath_list
# from utool import util_regex
# extended_regex_list, reflags = util_regex.extend_regex3(regex_list, reflags)
# if verbose:
# print('extended_regex_list = %r' % (extended_regex_list,))
# print('reflags = %r' % (reflags,))
_exprs_flags = [util_regex.extend_regex2(expr, reflags)
for expr in regex_list]
extended_regex_list = util_list.take_column(_exprs_flags, 0)
reflags_list = util_list.take_column(_exprs_flags, 1)
# HACK
reflags = reflags_list[0]
# For each matching filepath
for fpath in fpath_generator:
# For each search pattern
found_lines, found_lxs = grepfile(fpath, extended_regex_list,
reflags_list, cache=cache)
if inverse:
if len(found_lines) == 0:
# Append files that the pattern was not found in
found_fpath_list.append(fpath)
found_lines_list.append([])
found_lxs_list.append([])
elif len(found_lines) > 0:
found_fpath_list.append(fpath) # regular matching
found_lines_list.append(found_lines)
found_lxs_list.append(found_lxs)
grep_result = (found_fpath_list, found_lines_list, found_lxs_list)
if verbose:
print('==========')
print('==========')
print('[util_path] found matches in %d files' %
len(found_fpath_list))
print(make_grep_resultstr(grep_result, extended_regex_list, reflags))
# print('[util_path] found matches in %d files' % len(found_fpath_list))
# pat = util_regex.regex_or(extended_regex_list)
# for fpath, found, lxs in zip(found_fpath_list, found_lines_list,
# found_lxs_list):
# if len(found) > 0:
# print('----------------------')
# print('Found %d line(s) in %r: ' % (len(found), fpath))
# name = split(fpath)[1]
# max_line = len(lxs)
# ndigits = str(len(str(max_line)))
# fmt_str = '%s : %' + ndigits + 'd |%s'
# for (lx, line) in zip(lxs, found):
# # hack
# colored_line = util_str.highlight_regex(
# line.rstrip('\n'), pat, reflags=reflags)
# print(fmt_str % (name, lx, colored_line))
#print('[util_path] found matches in %d files' % len(found_fpath_list))
return grep_result
[docs]def make_grep_resultstr(grep_result, extended_regex_list, reflags, colored=True):
from utool import util_regex
from utool import util_str
msg_list = []
print_ = msg_list.append
pat = util_regex.regex_or(extended_regex_list)
found_fpath_list, found_lines_list, found_lxs_list = grep_result
for fpath, found, lxs in zip(found_fpath_list, found_lines_list,
found_lxs_list):
if len(found) > 0:
print_('----------------------')
print_('Found %d line(s) in %r: ' % (len(found), fpath))
name = split(fpath)[1]
max_line = len(lxs)
ndigits = str(len(str(max_line)))
fmt_str = '%s : %' + ndigits + 'd |%s'
for (lx, line) in zip(lxs, found):
# hack
colored_line = line.rstrip('\n')
if colored:
colored_line = util_str.highlight_regex(colored_line, pat, reflags=reflags)
print_(fmt_str % (name, lx, colored_line))
return '\n'.join(msg_list)
[docs]def get_win32_short_path_name(long_name):
"""
Gets the short path name of a given long path.
References:
http://stackoverflow.com/a/23598461/200291
http://stackoverflow.com/questions/23598289/get-win-short-fname-python
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> import utool as ut # NOQA
>>> # build test data
>>> #long_name = unicode(normpath(ut.get_resource_dir()))
>>> long_name = unicode(r'C:/Program Files (x86)')
>>> #long_name = unicode(r'C:/Python27')
#unicode(normpath(ut.get_resource_dir()))
>>> # execute function
>>> result = get_win32_short_path_name(long_name)
>>> # verify results
>>> print(result)
C:/PROGRA~2
"""
import ctypes
from ctypes import wintypes
_GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW
_GetShortPathNameW.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD]
_GetShortPathNameW.restype = wintypes.DWORD
output_buf_size = 0
while True:
output_buf = ctypes.create_unicode_buffer(output_buf_size)
needed = _GetShortPathNameW(long_name, output_buf, output_buf_size)
if output_buf_size >= needed:
short_name = output_buf.value
break
else:
output_buf_size = needed
return short_name
[docs]def expand_win32_shortname(path1):
try:
#try:
# import win32file
# path2 = win32file.GetLongPathName(path1)
#except ImportError:
import ctypes
#import win32file
path1 = unicode(path1)
buflen = 260 # max size
buf = ctypes.create_unicode_buffer(buflen)
ctypes.windll.kernel32.GetLongPathNameW(path1, buf, buflen)
# If the path doesnt exist windows doesnt return anything
path2 = buf.value if len(buf.value) > 0 else path1
except Exception as ex:
print(ex)
util_dbg.printex(ex, 'cannot fix win32 shortcut', keys=['path1', 'path2'])
path2 = path1
#raise
return path2
[docs]def existing_subpath(root_path, valid_subpaths, tiebreaker='first',
verbose=VERYVERBOSE):
"""
Returns join(root_path, subpath) where subpath in valid_subpath ane
exists(subpath)
"""
# Find the oxford_style groundtruth directory
for subpath in valid_subpaths:
path = join(root_path, subpath)
if checkpath(path, verbose=verbose):
if tiebreaker == 'first':
return path
raise AssertionError('none of the following subpaths exist: %r' %
(valid_subpaths,))
[docs]def existing_commonprefix(paths):
return longest_existing_path(commonprefix(paths))
#def find_executable(exename):
# import utool as ut
# search_dpaths = ut.get_install_dirs()
# pass
[docs]def search_in_dirs(fname, search_dpaths=[], shortcircuit=True,
return_tried=False, strict=False):
"""
search_in_dirs
Args:
fname (str): file name
search_dpaths (list):
shortcircuit (bool):
return_tried (bool): return tried paths
strict (bool): (default = False)
Returns:
fpath: None
Example:
>>> import utool as ut
>>> fname = 'Inno Setup 5\ISCC.exe'
>>> search_dpaths = ut.get_install_dirs()
>>> shortcircuit = True
>>> fpath = ut.search_in_dirs(fname, search_dpaths, shortcircuit)
>>> print(fpath)
"""
fpath_list = []
tried_list = []
for dpath in search_dpaths:
fpath = join(dpath, fname)
if return_tried:
tried_list.append(fpath)
if exists(fpath):
if shortcircuit:
if return_tried:
return fpath, tried_list
return fpath
else:
fpath_list.append(fpath)
if strict and len(fpath_list) == 0:
msg = ('Cannot find: fname=%r\n' % (fname,))
if return_tried:
msg += 'Tried: \n ' + '\n '.join(tried_list)
raise Exception(msg)
if shortcircuit:
if return_tried:
return None, tried_list
return None
else:
if return_tried:
return fpath_list, tried_list
return fpath_list
[docs]def find_lib_fpath(libname, root_dir, recurse_down=True, verbose=False, debug=False):
""" Search for the library """
def get_lib_fname_list(libname):
"""
input <libname>: library name (e.g. 'hesaff', not 'libhesaff')
returns <libnames>: list of plausible library file names
"""
if sys.platform.startswith('win32'):
libnames = ['lib' + libname + '.dll', libname + '.dll']
elif sys.platform.startswith('darwin'):
libnames = ['lib' + libname + '.dylib']
elif sys.platform.startswith('linux'):
libnames = ['lib' + libname + '.so']
else:
raise Exception('Unknown operating system: %s' % sys.platform)
return libnames
def get_lib_dpath_list(root_dir):
"""
input <root_dir>: deepest directory to look for a library (dll, so, dylib)
returns <libnames>: list of plausible directories to look.
"""
'returns possible lib locations'
get_lib_dpath_list = [root_dir,
join(root_dir, 'lib'),
join(root_dir, 'build'),
join(root_dir, 'build', 'lib')]
return get_lib_dpath_list
lib_fname_list = get_lib_fname_list(libname)
tried_fpaths = []
while root_dir is not None:
for lib_fname in lib_fname_list:
for lib_dpath in get_lib_dpath_list(root_dir):
lib_fpath = normpath(join(lib_dpath, lib_fname))
if exists(lib_fpath):
if verbose:
print('\n[c] Checked: '.join(tried_fpaths))
if debug:
print('using: %r' % lib_fpath)
return lib_fpath
else:
# Remember which candiate library fpaths did not exist
tried_fpaths.append(lib_fpath)
_new_root = dirname(root_dir)
if _new_root == root_dir:
root_dir = None
break
else:
root_dir = _new_root
if not recurse_down:
break
msg = ('\n[C!] load_clib(libname=%r root_dir=%r, recurse_down=%r, verbose=%r)' %
(libname, root_dir, recurse_down, verbose) +
'\n[c!] Cannot FIND dynamic library')
print(msg)
print('\n[c!] Checked: '.join(tried_fpaths))
raise ImportError(msg)
[docs]def ensure_mingw_drive(win32_path):
r""" replaces windows drives with mingw style drives
Args:
win32_path (str):
CommandLine:
python -m utool.util_path --test-ensure_mingw_drive
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> win32_path = r'C:/Program Files/Foobar'
>>> result = ensure_mingw_drive(win32_path)
>>> print(result)
/c/Program Files/Foobar
"""
win32_drive, _path = splitdrive(win32_path)
mingw_drive = '/' + win32_drive[:-1].lower()
mingw_path = mingw_drive + _path
return mingw_path
[docs]class ChdirContext(object):
"""
References http://www.astropython.org/snippet/2009/10/chdir-context-manager
"""
def __init__(self, dpath=None, stay=False):
self.stay = stay
self.curdir = os.getcwd()
self.dpath = dpath
def __enter__(self):
if self.dpath is not None:
print('[path.push] Change directory to %r' % (self.dpath,))
os.chdir(self.dpath)
return self
def __exit__(self, type_, value, trace):
if not self.stay:
print('[path.pop] Change directory to %r' % (self.curdir,))
os.chdir(self.curdir)
if trace is not None:
if VERBOSE:
print('[util_path] Error in chdir context manager!: ' + str(value))
return False # return a falsey value on error
[docs]def search_candidate_paths(candidate_path_list, candidate_name_list=None,
priority_paths=None, required_subpaths=[],
verbose=not QUIET):
"""
searches for existing paths that meed a requirement
Args:
candidate_path_list (list): list of paths to check. If
candidate_name_list is specified this is the dpath list instead
candidate_name_list (list): specifies several names to check
(default = None)
priority_paths (None): specifies paths to check first.
Ignore candidate_name_list (default = None)
required_subpaths (list): specified required directory structure
(default = [])
verbose (bool): verbosity flag(default = True)
Returns:
str: return_path
CommandLine:
python -m utool.util_path --test-search_candidate_paths
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_path import * # NOQA
>>> candidate_path_list = [ut.truepath('~/RPI/code/utool'),
>>> ut.truepath('~/code/utool')]
>>> candidate_name_list = None
>>> required_subpaths = []
>>> verbose = True
>>> priority_paths = None
>>> return_path = search_candidate_paths(candidate_path_list,
>>> candidate_name_list,
>>> priority_paths, required_subpaths,
>>> verbose)
>>> result = ('return_path = %s' % (str(return_path),))
>>> print(result)
"""
print('[search_candidate_paths] Searching for candidate paths')
import utool as ut
if candidate_name_list is not None:
candidate_path_list_ = [join(dpath, fname) for dpath, fname in
itertools.product(candidate_path_list,
candidate_name_list)]
else:
candidate_path_list_ = candidate_path_list
if priority_paths is not None:
candidate_path_list_ = priority_paths + candidate_path_list_
return_path = None
for path in candidate_path_list_:
if path is not None and exists(path):
if verbose:
print('[search_candidate_paths] Found candidate directory %r' % (path,))
print('[search_candidate_paths] ... checking for approprate structure')
# tomcat directory exists. Make sure it also contains a webapps dir
subpath_list = [join(path, subpath) for subpath in required_subpaths]
if all(ut.checkpath(path_, verbose=verbose) for path_ in subpath_list):
return_path = path
if verbose:
print('[search_candidate_paths] Found acceptable path')
return return_path
break
print('[search_candidate_paths] Failed to find acceptable path')
return return_path
if __name__ == '__main__':
"""
CommandLine:
python -c "import utool, utool.util_path; utool.doctest_funcs(utool.util_path)"
python -m utool.util_path
python -m utool.util_path --allexamples
python -m utool.util_path --allexamples --noface --nosrc
"""
import multiprocessing
multiprocessing.freeze_support() # for win32
import utool as ut # NOQA
ut.doctest_funcs()