<NME> util.py
<BEF> #!/usr/bin/env python
# -*- encoding: utf-8 -*-
r"""
Utility functions
-----------------

.. autosummary::
    :toctree: generated/

    import_lab
    expand_filepaths
    smkdirs
    filebase
    find_with_extension
"""

import os
import glob
import pandas as pd

from . import core


def import_lab(namespace, filename, infer_duration=True, **parse_options):
    r'''Load a .lab file as an Annotation object.

    .lab files are assumed to have the following format:

        ``TIME_START\tTIME_END\tANNOTATION``

    By default, .lab files are assumed to have columns separated by one
    or more white-space characters, and have no header or index column
    information.

    If the .lab file contains only two columns, then an empty duration
    field is inferred.

    If the .lab file contains more than three columns, each row's
    annotation value is assigned the contents of last non-empty column.


    Parameters
    ----------
    namespace : str
        The namespace for the new annotation

    filename : str
        Path to the .lab file

    infer_duration : bool
        If `True`, interval durations are inferred from `(start, end)` columns,
        or difference between successive times.

        If `False`, interval durations are assumed to be explicitly coded as
        `(start, duration)` columns.  If only one time column is given, then
        durations are set to 0.

        For instantaneous event annotations (e.g., beats or onsets), this
        should be set to `False`.

    parse_options : additional keyword arguments
        Passed to ``pandas.DataFrame.read_csv``

    Returns
    -------
    annotation : Annotation
        The newly constructed annotation object

    See Also
    --------
    pandas.DataFrame.read_csv
    '''

    # Create a new annotation object
    annotation = core.Annotation(namespace)

    parse_options.setdefault('sep', r'\s+')
    parse_options.setdefault('engine', 'python')
    parse_options.setdefault('header', None)
    parse_options.setdefault('index_col', False)

    # This is a hack to handle potentially ragged .lab data
    parse_options.setdefault('names', range(20))

    data = pd.read_csv(filename, **parse_options)

    # Drop all-nan columns
    data = data.dropna(how='all', axis=1)

    # Do we need to add a duration column?
    # This only applies to event annotations
    if len(data.columns) == 2:
        # Insert a column of zeros after the timing
        data.insert(1, 'duration', 0)
        if infer_duration:
            data['duration'][:-1] = data.loc[:, 0].diff()[1:].values

    else:
        # Convert from time to duration
        if infer_duration:
            data.loc[:, 1] -= data[0]

    for row in data.itertuples():
        time, duration = row[1:3]

        value = [x for x in row[3:] if x is not None][-1]

        annotation.append(time=time,
                          duration=duration,
                          confidence=1.0,
                          value=value)

    return annotation


def expand_filepaths(base_dir, rel_paths):
    """Expand a list of relative paths to a give base directory.

    Parameters
    ----------
    base_dir : str
        The target base directory

    rel_paths : list (or list-like)
        Collection of relative path strings

    Returns
    -------
    expanded_paths : list
        `rel_paths` rooted at `base_dir`

    Examples
    --------
    >>> jams.util.expand_filepaths('/data', ['audio', 'beat', 'seglab'])
    ['/data/audio', '/data/beat', '/data/seglab']

    """
    return [os.path.join(base_dir, os.path.normpath(rp)) for rp in rel_paths]


def smkdirs(dpath, mode=0o777):
    """Safely make a full directory path if it doesn't exist.

    Parameters
    ----------
    dpath : str
        Path of directory/directories to create

    mode : int [default=0777]
        Permissions for the new directories

    See also
    --------
    os.makedirs
    """
    if not os.path.exists(dpath):
        os.makedirs(dpath, mode=mode)


def filebase(filepath):
    """Return the extension-less basename of a file path.

    Parameters
    ----------
    filepath : str
        Path to a file

    Returns
    -------
    base : str
        The name of the file, with directory and extension removed

    Examples
    --------
    >>> jams.util.filebase('my_song.mp3')
    'my_song'

    """
    return os.path.splitext(os.path.basename(filepath))[0]


def find_with_extension(in_dir, ext, depth=3, sort=True):
    """Naive depth-search into a directory for files with a given extension.

    Parameters
    ----------
    in_dir : str
        Path to search.
    ext : str
        File extension to match.
    depth : int
        Depth of directories to search.
    sort : bool
        Sort the list alphabetically

    Returns
    -------
    matched : list
        return query(string)

    elif isinstance(query, six.string_types):
        return (re.match(query, string) is not None)

    else:
        raise TypeError('Invalid query type: {:s}'.format(type(query)))

    return False

     'Audio/Phoenix_ScotchMorris/Phoenix_ScotchMorris_STEMS/Phoenix_ScotchMorris_STEM_03.wav',
    'Audio/Phoenix_ScotchMorris/Phoenix_ScotchMorris_STEMS/Phoenix_ScotchMorris_STEM_04.wav']

    """
    assert depth >= 1
    ext = ext.strip(os.extsep)
    match = list()
    for n in range(1, depth+1):
        wildcard = os.path.sep.join(["*"]*n)
        search_path = os.path.join(in_dir, os.extsep.join([wildcard, ext]))
        match += glob.glob(search_path)

    if sort:
        match.sort()
    return match

<MSG> added test_match_query

<DFF> @@ -199,10 +199,10 @@ def match_query(string, query):
         return query(string)
 
     elif isinstance(query, six.string_types):
-        return (re.match(query, string) is not None)
+        return re.match(query, string) is not None
 
     else:
-        raise TypeError('Invalid query type: {:s}'.format(type(query)))
+        raise TypeError('Invalid query type: {}'.format(type(query)))
 
     return False
 

{'commit': "diff --git a/jams/tests/util_test.py b/jams/tests/util_test.py\nindex 1be4626..8975203 100644\n--- a/jams/tests/util_test.py\n+++ b/jams/tests/util_test.py\n@@ -4,7 +4,7 @@\n \n import tempfile\n import os\n-from nose.tools import eq_\n+from nose.tools import eq_, raises\n import numpy as np\n \n from jams import util\n@@ -67,3 +67,22 @@ def test_query_pop():\n     yield __test, 'alpha.beta.gamma', 'beta', '.', 'alpha.beta.gamma'\n     yield __test, 'alpha.beta.gamma', 'beta', '/', 'alpha.beta.gamma'\n     yield __test, 'alpha.alpha.beta.gamma', 'alpha', '.', 'alpha.beta.gamma'\n+\n+\n+def test_match_query():\n+\n+    def __test(needle, haystack, result):\n+        eq_(util.match_query(haystack, needle), result)\n+\n+    haystack = 'abcdeABCDE123'\n+\n+    yield __test, haystack, haystack, True\n+    yield __test, '.*cde.*', haystack, True\n+    yield __test, 'cde$', haystack, False\n+    yield __test, r'.*\\d+$', haystack, True\n+    yield __test, r'^\\d+$', haystack, False\n+\n+    yield __test, lambda x: True, haystack, True\n+    yield __test, lambda x: False, haystack, False\n+\n+    yield raises(TypeError)(__test), None, haystack, False\ndiff --git a/jams/util.py b/jams/util.py\nindex 3922732..903213e 100644\n--- a/jams/util.py\n+++ b/jams/util.py\n@@ -199,10 +199,10 @@ def match_query(string, query):\n         return query(string)\n \n     elif isinstance(query, six.string_types):\n-        return (re.match(query, string) is not None)\n+        return re.match(query, string) is not None\n \n     else:\n-        raise TypeError('Invalid query type: {:s}'.format(type(query)))\n+        raise TypeError('Invalid query type: {}'.format(type(query)))\n \n     return False\n \n", 'commit_subject': 'added test_match_query', 'repo_name': 'marl/jams', 'committer': {'name': 'Brian McFee', 'email': '90c31bc3c9e2d6ce7fc92adc83a94a44f88d935f@nyu.edu', 'time_sec': '1425598130', 'tz_offset': '-300', 'date': {'seconds': '1425598130'}}, 'commit_tree': 'f3c2717a247322493b1b71ab12688fa56ca12ee6', 'commit_message': 'added test_match_query\n', 'language_name': 'Shell', 'license': 'isc', 'hunks': "@@ -199,10 +199,10 @@ def match_query(string, query):\n         return query(string)\n \n     elif isinstance(query, six.string_types):\n-        return (re.match(query, string) is not None)\n+        return re.match(query, string) is not None\n \n     else:\n-        raise TypeError('Invalid query type: {:s}'.format(type(query)))\n+        raise TypeError('Invalid query type: {}'.format(type(query)))\n \n     return False\n \n", 'addition_count': 2, 'deletion_count': 2, 'src_file': 'a/jams/util.py', 'tgt_file': 'b/jams/util.py', 'file_extension': '.py', 'before_file': '#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\nr"""\nUtility functions\n-----------------\n\n.. autosummary::\n    :toctree: generated/\n\n    import_lab\n    expand_filepaths\n    smkdirs\n    filebase\n    find_with_extension\n"""\n\nimport os\nimport glob\nimport pandas as pd\n\nfrom . import core\n\n\ndef import_lab(namespace, filename, infer_duration=True, **parse_options):\n    r\'\'\'Load a .lab file as an Annotation object.\n\n    .lab files are assumed to have the following format:\n\n        ``TIME_START\\tTIME_END\\tANNOTATION``\n\n    By default, .lab files are assumed to have columns separated by one\n    or more white-space characters, and have no header or index column\n    information.\n\n    If the .lab file contains only two columns, then an empty duration\n    field is inferred.\n\n    If the .lab file contains more than three columns, each row\'s\n    annotation value is assigned the contents of last non-empty column.\n\n\n    Parameters\n    ----------\n    namespace : str\n        The namespace for the new annotation\n\n    filename : str\n        Path to the .lab file\n\n    infer_duration : bool\n        If `True`, interval durations are inferred from `(start, end)` columns,\n        or difference between successive times.\n\n        If `False`, interval durations are assumed to be explicitly coded as\n        `(start, duration)` columns.  If only one time column is given, then\n        durations are set to 0.\n\n        For instantaneous event annotations (e.g., beats or onsets), this\n        should be set to `False`.\n\n    parse_options : additional keyword arguments\n        Passed to ``pandas.DataFrame.read_csv``\n\n    Returns\n    -------\n    annotation : Annotation\n        The newly constructed annotation object\n\n    See Also\n    --------\n    pandas.DataFrame.read_csv\n    \'\'\'\n\n    # Create a new annotation object\n    annotation = core.Annotation(namespace)\n\n    parse_options.setdefault(\'sep\', r\'\\s+\')\n    parse_options.setdefault(\'engine\', \'python\')\n    parse_options.setdefault(\'header\', None)\n    parse_options.setdefault(\'index_col\', False)\n\n    # This is a hack to handle potentially ragged .lab data\n    parse_options.setdefault(\'names\', range(20))\n\n    data = pd.read_csv(filename, **parse_options)\n\n    # Drop all-nan columns\n    data = data.dropna(how=\'all\', axis=1)\n\n    # Do we need to add a duration column?\n    # This only applies to event annotations\n    if len(data.columns) == 2:\n        # Insert a column of zeros after the timing\n        data.insert(1, \'duration\', 0)\n        if infer_duration:\n            data[\'duration\'][:-1] = data.loc[:, 0].diff()[1:].values\n\n    else:\n        # Convert from time to duration\n        if infer_duration:\n            data.loc[:, 1] -= data[0]\n\n    for row in data.itertuples():\n        time, duration = row[1:3]\n\n        value = [x for x in row[3:] if x is not None][-1]\n\n        annotation.append(time=time,\n                          duration=duration,\n                          confidence=1.0,\n                          value=value)\n\n    return annotation\n\n\ndef expand_filepaths(base_dir, rel_paths):\n    """Expand a list of relative paths to a give base directory.\n\n    Parameters\n    ----------\n    base_dir : str\n        The target base directory\n\n    rel_paths : list (or list-like)\n        Collection of relative path strings\n\n    Returns\n    -------\n    expanded_paths : list\n        `rel_paths` rooted at `base_dir`\n\n    Examples\n    --------\n    >>> jams.util.expand_filepaths(\'/data\', [\'audio\', \'beat\', \'seglab\'])\n    [\'/data/audio\', \'/data/beat\', \'/data/seglab\']\n\n    """\n    return [os.path.join(base_dir, os.path.normpath(rp)) for rp in rel_paths]\n\n\ndef smkdirs(dpath, mode=0o777):\n    """Safely make a full directory path if it doesn\'t exist.\n\n    Parameters\n    ----------\n    dpath : str\n        Path of directory/directories to create\n\n    mode : int [default=0777]\n        Permissions for the new directories\n\n    See also\n    --------\n    os.makedirs\n    """\n    if not os.path.exists(dpath):\n        os.makedirs(dpath, mode=mode)\n\n\ndef filebase(filepath):\n    """Return the extension-less basename of a file path.\n\n    Parameters\n    ----------\n    filepath : str\n        Path to a file\n\n    Returns\n    -------\n    base : str\n        The name of the file, with directory and extension removed\n\n    Examples\n    --------\n    >>> jams.util.filebase(\'my_song.mp3\')\n    \'my_song\'\n\n    """\n    return os.path.splitext(os.path.basename(filepath))[0]\n\n\ndef find_with_extension(in_dir, ext, depth=3, sort=True):\n    """Naive depth-search into a directory for files with a given extension.\n\n    Parameters\n    ----------\n    in_dir : str\n        Path to search.\n    ext : str\n        File extension to match.\n    depth : int\n        Depth of directories to search.\n    sort : bool\n        Sort the list alphabetically\n\n    Returns\n    -------\n    matched : list\n        return query(string)\n\n    elif isinstance(query, six.string_types):\n        return (re.match(query, string) is not None)\n\n    else:\n        raise TypeError(\'Invalid query type: {:s}\'.format(type(query)))\n\n    return False\n\n     \'Audio/Phoenix_ScotchMorris/Phoenix_ScotchMorris_STEMS/Phoenix_ScotchMorris_STEM_03.wav\',\n    \'Audio/Phoenix_ScotchMorris/Phoenix_ScotchMorris_STEMS/Phoenix_ScotchMorris_STEM_04.wav\']\n\n    """\n    assert depth >= 1\n    ext = ext.strip(os.extsep)\n    match = list()\n    for n in range(1, depth+1):\n        wildcard = os.path.sep.join(["*"]*n)\n        search_path = os.path.join(in_dir, os.extsep.join([wildcard, ext]))\n        match += glob.glob(search_path)\n\n    if sort:\n        match.sort()\n    return match\n', 'message': 'added test_match_query\n', 'author_name': 'Brian McFee', 'lang': 'py'}