<gh_stars>0
# -*- coding: utf-8 -*-
"""
Command line parsing and command dispatch
"""

from __future__ import print_function
from builtins import input      # pylint:disable=W0622
from datetime import datetime
from functools import partial
import hashlib
import json
import os
import platform
import re
from shutil import rmtree, copyfile
import socket
import stat
import subprocess
import sys
import tempfile
import time
import yaml

import numpy as np
from packaging.version import Version
import pandas as pd
import pkg_resources
import requests
from six import itervalues, string_types
from six.moves.urllib.parse import urlparse, urlunparse
from tqdm import tqdm

from .build import (build_package, build_package_from_contents, generate_build_file,
                    generate_contents, get_or_create_package, BuildException, load_yaml)
from .compat import pathlib
from .const import DEFAULT_BUILDFILE, DTIMEF, QuiltException, SYSTEM_METADATA, TargetType
from .core import (LATEST_TAG, GroupNode, RootNode, decode_node, encode_node,
                   find_object_hashes, hash_contents)
from .data_transfer import download_fragments, upload_fragments
from .store import PackageStore, StoreException
from .util import (BASE_DIR, gzip_compress, is_nodename, fs_link, parse_package as parse_package_util,
                   parse_package_extended as parse_package_extended_util)
from ..imports import _from_core_node

from .. import nodes


DEFAULT_REGISTRY_URL = 'https://pkg.quiltdata.com'
GIT_URL_RE = re.compile(r'(?P<url>http[s]?://[\w./~_-]+\.git)(?:@(?P<branch>[\w_-]+))?')

LOG_TIMEOUT = 3 # 3 seconds

VERSION = pkg_resources.require('quilt')[0].version


class CommandException(QuiltException):
    """
    Exception class for all command-related failures.
    """
    pass

class HTTPResponseException(CommandException):
    def __init__(self, message, response):
        super(HTTPResponseException, self).__init__(message)
        self.response = response

_registry_url = None

def parse_package_extended(identifier):
    #TODO: Unwrap this and modify 'util' version to raise QuiltException
    try:
        return parse_package_extended_util(identifier)
    except ValueError:
        pkg_format = '[team:]owner/package_name/path[:v:<version> or :t:<tag> or :h:<hash>]'
        raise CommandException("Specify package as %s." % pkg_format)

def parse_package(name, allow_subpath=False):
    #TODO: Unwrap this and modify 'util' version to raise QuiltException and call check_name()
    try:
        if allow_subpath:
            team, owner, pkg, subpath = parse_package_util(name, allow_subpath)
        else:
            team, owner, pkg = parse_package_util(name, allow_subpath)
            subpath = None
    except ValueError:
        pkg_format = '[team:]owner/package_name/path' if allow_subpath else '[team:]owner/package_name'
        raise CommandException("Specify package as %s." % pkg_format)

    try:
        PackageStore.check_name(team, owner, pkg, subpath)
    except StoreException as ex:
        raise CommandException(str(ex))

    if allow_subpath:
        return team, owner, pkg, subpath
    return team, owner, pkg


def _load_config():
    config_path = os.path.join(BASE_DIR, 'config.json')
    if os.path.exists(config_path):
        with open(config_path) as fd:
            return json.load(fd)
    return {}

def _save_config(cfg):
    if not os.path.exists(BASE_DIR):
        os.makedirs(BASE_DIR)
    config_path = os.path.join(BASE_DIR, 'config.json')
    with open(config_path, 'w') as fd:
        json.dump(cfg, fd)

def _load_auth():
    auth_path = os.path.join(BASE_DIR, 'auth.json')
    if os.path.exists(auth_path):
        with open(auth_path) as fd:
            auth = json.load(fd)
            if 'access_token' in auth:
                # Old format; ignore it.
                auth = {}
            return auth
    return {}

def _save_auth(cfg):
    if not os.path.exists(BASE_DIR):
        os.makedirs(BASE_DIR)
    auth_path = os.path.join(BASE_DIR, 'auth.json')
    with open(auth_path, 'w') as fd:
        os.chmod(auth_path, stat.S_IRUSR | stat.S_IWUSR)
        json.dump(cfg, fd)

def get_registry_url(team):
    if team is not None:
        return "https://%s-registry.team.quiltdata.com" % team

    global _registry_url
    if _registry_url is not None:
        return _registry_url

    # Env variable; overrides the config.
    url = os.environ.get('QUILT_PKG_URL')
    if url is None:
        # Config file (generated by `quilt config`).
        cfg = _load_config()
        url = cfg.get('registry_url', '')

    # '' means default URL.
    _registry_url = url or DEFAULT_REGISTRY_URL
    return _registry_url

def config():
    answer = input("Please enter the URL for your custom Quilt registry (ask your administrator),\n"
                   "or leave this line blank to use the default registry: ")
    if answer:
        url = urlparse(answer.rstrip('/'))
        if (url.scheme not in ['http', 'https'] or not url.netloc or
            url.path or url.params or url.query or url.fragment):
            raise CommandException("Invalid URL: %s" % answer)
        canonical_url = urlunparse(url)
    else:
        # When saving the config, store '' instead of the actual URL in case we ever change it.
        canonical_url = ''

    cfg = _load_config()
    cfg['registry_url'] = canonical_url
    _save_config(cfg)

    # Clear the cached URL.
    global _registry_url
    _registry_url = None

def _update_auth(team, refresh_token, timeout=None):
    response = requests.post("%s/api/token" % get_registry_url(team),
        timeout=timeout,
        data=dict(
            refresh_token=refresh_token,
        ))

    if response.status_code != requests.codes.ok:
        raise CommandException("Authentication error: %s" % response.status_code)

    data = response.json()
    error = data.get('error')
    if error is not None:
        raise CommandException("Failed to log in: %s" % error)

    return dict(
        team=team,
        refresh_token=data['refresh_token'],
        access_token=data['access_token'],
        expires_at=data['expires_at']
    )

def _handle_response(team, resp, **kwargs):
    _ = kwargs                  # unused    pylint:disable=W0613
    if resp.status_code == requests.codes.unauthorized:
        raise CommandException(
            "Authentication failed. Run `quilt login%s` again." %
            (' ' + team if team else '')
        )
    elif not resp.ok:
        try:
            data = resp.json()
            raise HTTPResponseException(data['message'], resp)
        except ValueError:
            raise HTTPResponseException("Unexpected failure: error %s" % resp.status_code, resp)

def _create_auth(team, timeout=None):
    """
    Reads the credentials, updates the access token if necessary, and returns it.
    """
    url = get_registry_url(team)
    contents = _load_auth()
    auth = contents.get(url)

    if auth is not None:
        # If the access token expires within a minute, update it.
        if auth['expires_at'] < time.time() + 60:
            try:
                auth = _update_auth(team, auth['refresh_token'], timeout)
            except CommandException as ex:
                raise CommandException(
                    "Failed to update the access token (%s). Run `quilt login%s` again." %
                    (ex, ' ' + team if team else '')
                )
            contents[url] = auth
            _save_auth(contents)

    return auth

def _create_session(team, auth):
    """
    Creates a session object to be used for `push`, `install`, etc.
    """
    session = requests.Session()
    session.hooks.update(dict(
        response=partial(_handle_response, team)
    ))
    session.headers.update({
        "Content-Type": "application/json",
        "Accept": "application/json",
        "User-Agent": "quilt-cli/%s (%s %s) %s/%s" % (
            VERSION, platform.system(), platform.release(),
            platform.python_implementation(), platform.python_version()
        )
    })
    if auth is not None:
        session.headers["Authorization"] = "Bearer %s" % auth['access_token']

    return session

_sessions = {}                  # pylint:disable=C0103

def _get_session(team, timeout=None):
    """
    Creates a session or returns an existing session.
    """
    global _sessions            # pylint:disable=C0103
    session = _sessions.get(team)
    if session is None:
        auth = _create_auth(team, timeout)
        _sessions[team] = session = _create_session(team, auth)

    assert session is not None

    return session

def _clear_session(team):
    global _sessions            # pylint:disable=C0103
    session = _sessions.pop(team, None)
    if session is not None:
        session.close()

def _open_url(url):
    try:
        if sys.platform == 'win32':
            os.startfile(url)   # pylint:disable=E1101
        elif sys.platform == 'darwin':
            with open(os.devnull, 'r+') as null:
                subprocess.check_call(['open', url], stdin=null, stdout=null, stderr=null)
        else:
            with open(os.devnull, 'r+') as null:
                subprocess.check_call(['xdg-open', url], stdin=null, stdout=null, stderr=null)
    except Exception as ex:     # pylint:disable=W0703
        print("Failed to launch the browser: %s" % ex)

def _match_hash(package, hash):
    team, owner, pkg, _ = parse_package(package, allow_subpath=True)
    session = _get_session(team)

    hash = hash.lower()

    if not (6 <= len(hash) <= 64):
        raise CommandException('Invalid hash of length {}: {!r}\n  '
                               'Ensure that the hash is between 6 and 64 characters.'
                               .format(len(hash), hash))

    # short-circuit for exact length
    if len(hash) == 64:
        return hash

    response = session.get(
        "{url}/api/log/{owner}/{pkg}/".format(
            url=get_registry_url(team),
            owner=owner,
            pkg=pkg
        )
    )

    matches = set(entry['hash'] for entry in response.json()['logs']
                  if entry['hash'].startswith(hash))

    if len(matches) == 1:
        return matches.pop()
    if len(matches) > 1:
        # Sorting for consistency in testing, as well as visual comparison of hashes
        ambiguous = '\n'.join(sorted(matches))
        raise CommandException(
            "Ambiguous hash for package {package}: {hash!r} matches the folowing hashes:\n\n{ambiguous}"
            .format(package=package, hash=hash, ambiguous=ambiguous))
    raise CommandException("Invalid hash for package {package}: {hash}".format(package=package, hash=hash))

def _find_logged_in_team():
    """
    Find a team name in the auth credentials.
    There should be at most one, since we don't allow multiple team logins.
    """
    contents = _load_auth()
    auth = next(itervalues(contents), {})
    return auth.get('team')

def _check_team_login(team):
    """
    Disallow simultaneous public cloud and team logins.
    """
    contents = _load_auth()

    for auth in itervalues(contents):
        existing_team = auth.get('team')
        if team and team != existing_team:
            raise CommandException(
                "Can't log in as team %r; log out first." % team
            )
        elif not team and existing_team:
            raise CommandException(
                "Can't log in as a public user; log out from team %r first." % existing_team
            )

team_regex = re.compile('^[a-z]+$')
def _check_team_id(team):
    if team is not None and team_regex.match(team) is None:
        raise CommandException(
            "Invalid team name: {team}. Lowercase letters only.".format(team=team)
            )

def _check_team_exists(team):
    """
    Check that the team registry actually exists.
    """
    if team is None:
        return

    hostname = '%s-registry.team.quiltdata.com' % team
    try:
        socket.gethostbyname(hostname)
    except IOError:
        try:
            # Do we have internet?
            socket.gethostbyname('quiltdata.com')
        except IOError:
            message = "Can't find quiltdata.com. Check your internet connection."
        else:
            message = "Unable to connect to registry. Is the team name %r correct?" % team
        raise CommandException(message)

def login(team=None):
    """
    Authenticate.

    Launches a web browser and asks the user for a token.
    """
    _check_team_id(team)
    _check_team_exists(team)
    _check_team_login(team)

    login_url = "%s/login" % get_registry_url(team)

    print("Launching a web browser...")
    print("If that didn't work, please visit the following URL: %s" % login_url)

    _open_url(login_url)

    print()
    refresh_token = input("Enter the code from the webpage: ")

    login_with_token(refresh_token, team)

def login_with_token(refresh_token, team=None):
    """
    Authenticate using an existing token.
    """
    # Get an access token and a new refresh token.
    _check_team_id(team)
    auth = _update_auth(team, refresh_token)

    url = get_registry_url(team)
    contents = _load_auth()
    contents[url] = auth
    _save_auth(contents)

    _clear_session(team)

def logout():
    """
    Become anonymous. Useful for testing.
    """
    # TODO revoke refresh token (without logging out of web sessions)
    if _load_auth():
        _save_auth({})
    else:
        print("Already logged out.")

    global _sessions            # pylint:disable=C0103
    _sessions = {}

def generate(directory, outfilename=DEFAULT_BUILDFILE):
    """
    Generate a build-file for quilt build from a directory of
    source files.
    """
    try:
        buildfilepath = generate_build_file(directory, outfilename=outfilename)
    except BuildException as builderror:
        raise CommandException(str(builderror))

    print("Generated build-file %s." % (buildfilepath))

def check(path=None, env='default'):
    """
    Execute the checks: rules for a given build.yml file.
    """
    # TODO: add files=<list of files> to check only a subset...
    # also useful for 'quilt build' to exclude certain files?
    # (if not, then require dry_run=True if files!=None/all)
    build("dry_run/dry_run", path=path, dry_run=True, env=env)

def _clone_git_repo(url, branch, dest):
    cmd = ['git', 'clone', '-q', '--depth=1']
    if branch:
        cmd += ['-b', branch]
    cmd += [url, dest]
    subprocess.check_call(cmd)

def _log(team, **kwargs):
    # TODO(dima): Save logs to a file, then send them when we get a chance.

    cfg = _load_config()
    if cfg.get('disable_analytics'):
        return

    session = None
    try:
        session = _get_session(team, timeout=LOG_TIMEOUT)

        # Disable error handling.
        orig_response_hooks = session.hooks.pop('response')

        session.post(
            "{url}/api/log".format(
                url=get_registry_url(team),
            ),
            data=json.dumps([kwargs]),
            timeout=LOG_TIMEOUT,
        )
    except requests.exceptions.RequestException:
        # Ignore logging errors.
        pass
    finally:
        # restore disabled error-handling
        if session:
            session.hooks['response'] = orig_response_hooks

def build(package, path=None, dry_run=False, env='default', force=False, build_file=False):
    """
    Compile a Quilt data package, either from a build file or an existing package node.

    :param package: short package specifier, i.e. 'team:user/pkg'
    :param path: file path, git url, or existing package node
    """
    # TODO: rename 'path' param to 'target'?
    team, _, _, subpath = parse_package(package, allow_subpath=True)
    _check_team_id(team)
    logged_in_team = _find_logged_in_team()
    if logged_in_team is not None and team is None and force is False:
        answer = input("You're logged in as a team member, but you aren't specifying "
                       "a team for the package you're currently building. Maybe you meant:\n"
                       "quilt build {team}:{package}\n"
                       "Are you sure you want to continue? (y/N) ".format(
                                team=logged_in_team, package=package))
        if answer.lower() != 'y':
            return

    # Backward compatibility: if there's no subpath, we're building a top-level package,
    # so treat `path` as a build file, not as a data node.
    if not subpath:
        build_file = True

    package_hash = hashlib.md5(package.encode('utf-8')).hexdigest()
    try:
        _build_internal(package, path, dry_run, env, build_file)
    except Exception as ex:
        _log(team, type='build', package=package_hash, dry_run=dry_run, env=env, error=str(ex))
        raise
    _log(team, type='build', package=package_hash, dry_run=dry_run, env=env)

def _build_internal(package, path, dry_run, env, build_file):
    if build_file and isinstance(path, string_types):
        # is this a git url?
        is_git_url = GIT_URL_RE.match(path)
        if is_git_url:
            tmpdir = tempfile.mkdtemp()
            url = is_git_url.group('url')
            branch = is_git_url.group('branch')
            try:
                _clone_git_repo(url, branch, tmpdir)
                build_from_path(package, tmpdir, dry_run=dry_run, env=env)
            except Exception as exc:
                msg = "attempting git clone raised exception: {exc}"
                raise CommandException(msg.format(exc=exc))
            finally:
                if os.path.exists(tmpdir):
                    rmtree(tmpdir)
        else:
            build_from_path(package, path, dry_run=dry_run, env=env)
    elif isinstance(path, nodes.Node):
        assert not dry_run  # TODO?
        build_from_node(package, path)
    elif isinstance(path, string_types + (pd.DataFrame, np.ndarray)):
        assert not dry_run  # TODO?
        build_from_node(package, nodes.DataNode(None, None, path, {}))
    elif path is None:
        assert not dry_run  # TODO?
        build_from_node(package, nodes.GroupNode({}))
    else:
        raise ValueError("Expected a GroupNode, path, git URL, DataFrame, ndarray, or None, but got %r" % path)


def build_from_node(package, node):
    """
    Compile a Quilt data package from an existing package node.
    """
    team, owner, pkg, subpath = parse_package(package, allow_subpath=True)
    _check_team_id(team)
    store = PackageStore()

    pkg_root = get_or_create_package(store, team, owner, pkg, subpath)

    if not subpath and not isinstance(node, nodes.GroupNode):
        raise CommandException("Top-level node must be a group")

    def _process_node(node, path):
        if not isinstance(node._meta, dict):
            raise CommandException(
                "Error in %s: value must be a dictionary" % '.'.join(path + ['_meta'])
            )
        meta = dict(node._meta)
        system_meta = meta.pop(SYSTEM_METADATA, {})
        if not isinstance(system_meta, dict):
            raise CommandException(
                "Error in %s: %s overwritten. %s is a reserved metadata key. Try a different key." %
                ('.'.join(path + ['_meta']), SYSTEM_METADATA, SYSTEM_METADATA)
            )
        if isinstance(node, nodes.GroupNode):
            store.add_to_package_group(pkg_root, path, meta)
            for key, child in node._items():
                _process_node(child, path + [key])
        elif isinstance(node, nodes.DataNode):
            # TODO: Reuse existing fragments if we have them.
            data = node._data()
            filepath = system_meta.get('filepath')
            transform = system_meta.get('transform')
            if isinstance(data, pd.DataFrame):
                store.add_to_package_df(pkg_root, data, path, TargetType.PANDAS, filepath, transform, meta)
            elif isinstance(data, np.ndarray):
                store.add_to_package_numpy(pkg_root, data, path, TargetType.NUMPY, filepath, transform, meta)
            elif isinstance(data, string_types):
                store.add_to_package_file(pkg_root, data, path, TargetType.FILE, filepath, transform, meta)
            else:
                assert False, "Unexpected data type: %r" % data
        else:
            assert False, "Unexpected node type: %r" % node

    try:
        _process_node(node, subpath)
    except StoreException as ex:
        raise CommandException("Failed to build the package: %s" % ex)

    store.save_package_contents(pkg_root, team, owner, pkg)

def build_from_path(package, path, dry_run=False, env='default', outfilename=DEFAULT_BUILDFILE):
    """
    Compile a Quilt data package from a build file.
    Path can be a directory, in which case the build file will be generated automatically.
    """
    team, owner, pkg, subpath = parse_package(package, allow_subpath=True)

    if not os.path.exists(path):
        raise CommandException("%s does not exist." % path)

    try:
        if os.path.isdir(path):
            buildpath = os.path.join(path, outfilename)
            if os.path.exists(buildpath):
                raise CommandException(
                    "Build file already exists. Run `quilt build %r` instead." % buildpath
                )

            contents = generate_contents(path, outfilename)
            build_package_from_contents(team, owner, pkg, subpath, path, contents, dry_run=dry_run, env=env)
        else:
            build_package(team, owner, pkg, subpath, path, dry_run=dry_run, env=env)

        if not dry_run:
            print("Built %s successfully." % package)
    except BuildException as ex:
        raise CommandException("Failed to build the package: %s" % ex)

def log(package):
    """
    List all of the changes to a package on the server.
    """
    team, owner, pkg = parse_package(package)
    session = _get_session(team)

    response = session.get(
        "{url}/api/log/{owner}/{pkg}/".format(
            url=get_registry_url(team),
            owner=owner,
            pkg=pkg
        )
    )

    table = [("Hash", "Pushed", "Author", "Tags", "Versions")]
    for entry in reversed(response.json()['logs']):
        ugly = datetime.fromtimestamp(entry['created'])
        nice = ugly.strftime("%Y-%m-%d %H:%M:%S")
        table.append((entry['hash'], nice, entry['author'],
            str(entry.get('tags', [])), str(entry.get('versions', []))))
    _print_table(table)

def push(package, is_public=False, is_team=False, reupload=False):
    """
    Push a Quilt data package to the server
    """
    team, owner, pkg, subpath = parse_package(package, allow_subpath=True)
    _check_team_id(team)
    session = _get_session(team)

    store, pkgroot = PackageStore.find_package(team, owner, pkg)
    if pkgroot is None:
        raise CommandException("Package {package} not found.".format(package=package))

    pkghash = hash_contents(pkgroot)
    contents = pkgroot

    for component in subpath:
        try:
            contents = contents.children[component]
        except (AttributeError, KeyError):
            raise CommandException("Invalid subpath: %r" % component)

    def _push_package(dry_run=False, sizes=dict()):
        data = json.dumps(dict(
            dry_run=dry_run,
            is_public=is_public,
            is_team=is_team,
            contents=contents,
            description="",  # TODO
            sizes=sizes
        ), default=encode_node)

        compressed_data = gzip_compress(data.encode('utf-8'))

        if subpath:
            return session.post(
                "{url}/api/package_update/{owner}/{pkg}/{subpath}".format(
                    url=get_registry_url(team),
                    owner=owner,
                    pkg=pkg,
                    subpath='/'.join(subpath)
                ),
                data=compressed_data,
                headers={
                    'Content-Encoding': 'gzip'
                }
            )
        else:
            return session.put(
                "{url}/api/package/{owner}/{pkg}/{hash}".format(
                    url=get_registry_url(team),
                    owner=owner,
                    pkg=pkg,
                    hash=pkghash
                ),
                data=compressed_data,
                headers={
                    'Content-Encoding': 'gzip'
                }
            )

    print("Fetching upload URLs from the registry...")
    resp = _push_package(dry_run=True)
    obj_urls = resp.json()['upload_urls']

    assert set(obj_urls) == set(find_object_hashes(contents))

    obj_sizes = {
        obj_hash: os.path.getsize(store.object_path(obj_hash)) for obj_hash in obj_urls
    }

    success = upload_fragments(store, obj_urls, obj_sizes, reupload=reupload)
    if not success:
        raise CommandException("Failed to upload fragments")

    print("Uploading package metadata...")
    resp = _push_package(sizes=obj_sizes)
    package_url = resp.json()['package_url']

    if not subpath:
        # Update the latest tag.
        print("Updating the 'latest' tag...")
        session.put(
            "{url}/api/tag/{owner}/{pkg}/{tag}".format(
                url=get_registry_url(team),
                owner=owner,
                pkg=pkg,
                tag=LATEST_TAG
            ),
            data=json.dumps(dict(
                hash=pkghash
            ))
        )

    print("Push complete. %s is live:\n%s" % (package, package_url))

def version_list(package):
    """
    List the versions of a package.
    """
    team, owner, pkg = parse_package(package)
    session = _get_session(team)

    response = session.get(
        "{url}/api/version/{owner}/{pkg}/".format(
            url=get_registry_url(team),
            owner=owner,
            pkg=pkg
        )
    )

    for version in response.json()['versions']:
        print("%s: %s" % (version['version'], version['hash']))

def version_add(package, version, pkghash, force=False):
    """
    Add a new version for a given package hash.

    Version format needs to follow PEP 440.
    Versions are permanent - once created, they cannot be modified or deleted.
    """
    team, owner, pkg = parse_package(package)
    session = _get_session(team)

    try:
        Version(version)
    except ValueError:
        url = "https://www.python.org/dev/peps/pep-0440/#examples-of-compliant-version-schemes"
        raise CommandException(
            "Invalid version format; see %s" % url
        )

    if not force:
        answer = input("Versions cannot be modified or deleted; are you sure? (y/n) ")
        if answer.lower() != 'y':
            return

    session.put(
        "{url}/api/version/{owner}/{pkg}/{version}".format(
            url=get_registry_url(team),
            owner=owner,
            pkg=pkg,
            version=version
        ),
        data=json.dumps(dict(
            hash=_match_hash(package, pkghash)
        ))
    )

def tag_list(package):
    """
    List the tags of a package.
    """
    team, owner, pkg = parse_package(package)
    session = _get_session(team)

    response = session.get(
        "{url}/api/tag/{owner}/{pkg}/".format(
            url=get_registry_url(team),
            owner=owner,
            pkg=pkg
        )
    )

    for tag in response.json()['tags']:
        print("%s: %s" % (tag['tag'], tag['hash']))

def tag_add(package, tag, pkghash):
    """
    Add a new tag for a given package hash.

    Unlike versions, tags can have an arbitrary format, and can be modified
    and deleted.

    When a package is pushed, it gets the "latest" tag.
    """
    team, owner, pkg = parse_package(package)
    session = _get_session(team)

    session.put(
        "{url}/api/tag/{owner}/{pkg}/{tag}".format(
            url=get_registry_url(team),
            owner=owner,
            pkg=pkg,
            tag=tag
        ),
        data=json.dumps(dict(
            hash=_match_hash(package, pkghash)
        ))
    )

def tag_remove(package, tag):
    """
    Delete a tag.
    """
    team, owner, pkg = parse_package(package)
    session = _get_session(team)

    session.delete(
        "{url}/api/tag/{owner}/{pkg}/{tag}".format(
            url=get_registry_url(team),
            owner=owner,
            pkg=pkg,
            tag=tag
        )
    )

def install_via_requirements(requirements_str, force=False):
    """
    Download multiple Quilt data packages via quilt.xml requirements file.
    """
    if requirements_str[0] == '@':
        path = requirements_str[1:]
        if os.path.isfile(path):
            yaml_data = load_yaml(path)
            if 'packages' not in yaml_data.keys():
                raise CommandException('Error in {filename}: missing "packages" node'.format(filename=path))
        else:
            raise CommandException("Requirements file not found: {filename}".format(filename=path))
    else:
        yaml_data = yaml.load(requirements_str)
    for pkginfo in yaml_data['packages']:
        info = parse_package_extended(pkginfo)
        install(info.full_name, info.hash, info.version, info.tag, force=force)

def install(package, hash=None, version=None, tag=None, force=False, meta_only=False):
    """
    Download a Quilt data package from the server and install locally.

    At most one of `hash`, `version`, or `tag` can be given. If none are
    given, `tag` defaults to "latest".

    `package` may be a node tree - in which case, its fragments get downloaded.
    No other parameters are allowed.
    """
    if isinstance(package, nodes.Node):
        if not (hash is version is tag is None and force is meta_only is False):
            raise ValueError("Parameters not allowed when installing a node")
        _materialize(package)
        return

    if hash is version is tag is None:
        tag = LATEST_TAG

    # @filename ==> read from file
    # newline = multiple lines ==> multiple requirements
    package = package.strip()
    if len(package) == 0:
        raise CommandException("package name is empty.")

    if package[0] == '@' or '\n' in package:
        return install_via_requirements(package, force=force)

    assert [hash, version, tag].count(None) == 2

    team, owner, pkg, subpath = parse_package(package, allow_subpath=True)
    _check_team_id(team)
    session = _get_session(team)
    store = PackageStore()
    existing_pkg = store.get_package(team, owner, pkg)

    print("Downloading package metadata...")

    try:
        if version is not None:
            response = session.get(
                "{url}/api/version/{owner}/{pkg}/{version}".format(
                    url=get_registry_url(team),
                    owner=owner,
                    pkg=pkg,
                    version=version
                )
            )
            pkghash = response.json()['hash']
        elif tag is not None:
            response = session.get(
                "{url}/api/tag/{owner}/{pkg}/{tag}".format(
                    url=get_registry_url(team),
                    owner=owner,
                    pkg=pkg,
                    tag=tag
                )
            )
            pkghash = response.json()['hash']
        else:
            pkghash = _match_hash(package, hash)
    except HTTPResponseException as e:
        logged_in_team = _find_logged_in_team()
        if (team is None and logged_in_team is not None
                and e.response.status_code == requests.codes.not_found):
            raise CommandException("Package {owner}/{pkg} does not exist. "
                                   "Maybe you meant {team}:{owner}/{pkg}?".format(
                                   owner=owner, pkg=pkg, team=logged_in_team))
        else:
            raise

    assert pkghash is not None

    response = session.get(
        "{url}/api/package/{owner}/{pkg}/{hash}".format(
            url=get_registry_url(team),
            owner=owner,
            pkg=pkg,
            hash=pkghash
        ),
        params=dict(
            subpath='/'.join(subpath),
            meta_only='true' if meta_only else ''
        )
    )
    assert response.ok # other responses handled by _handle_response

    if existing_pkg is not None and not force:
        print("{package} already installed.".format(package=package))
        overwrite = input("Overwrite? (y/n) ")
        if overwrite.lower() != 'y':
            return

    dataset = response.json(object_hook=decode_node)
    contents = dataset['contents']

    # Verify contents hash
    if pkghash != hash_contents(contents):
        raise CommandException("Mismatched hash. Try again.")

    # TODO: Shouldn't need this? At least don't need the contents
    store.install_package(team, owner, pkg, contents)

    obj_urls = dataset['urls']
    obj_sizes = dataset['sizes']

    # Skip the objects we already have
    for obj_hash in list(obj_urls):
        if os.path.exists(store.object_path(obj_hash)):
            del obj_urls[obj_hash]
            del obj_sizes[obj_hash]

    if obj_urls:
        success = download_fragments(store, obj_urls, obj_sizes)
        if not success:
            raise CommandException("Failed to download fragments")
    else:
        print("Fragments already downloaded")

    store.save_package_contents(contents, team, owner, pkg)

def _materialize(node):
    store = PackageStore()

    hashes = set()

    stack = [node]
    while stack:
        obj = stack.pop()
        if isinstance(obj, nodes.GroupNode):
            stack.extend(child for name, child in obj._items())
        else:
            hashes.update(obj._hashes or [])  # May be empty for nodes created locally

    missing_hashes = {obj_hash for obj_hash in hashes if not os.path.exists(store.object_path(obj_hash))}

    if missing_hashes:
        print("Requesting %d signed URLs..." % len(missing_hashes))

        teams = {None, _find_logged_in_team()}

        obj_urls = dict()
        obj_sizes = dict()

        for team in teams:
            session = _get_session(team)
            response = session.post(
                "{url}/api/get_objects".format(url=get_registry_url(team)),
                json=list(missing_hashes)
            )
            data = response.json()
            obj_urls.update(data['urls'])
            obj_sizes.update(data['sizes'])

        if len(obj_urls) != len(missing_hashes):
            not_found = sorted(missing_hashes - set(obj_urls))
            raise CommandException("Unable to download the following hashes: %s" % ', '.join(not_found))

        success = download_fragments(store, obj_urls, obj_sizes)
        if not success:
            raise CommandException("Failed to download fragments")
    else:
        print("Fragments already downloaded")

def access_list(package):
    """
    Print list of users who can access a package.
    """
    team, owner, pkg = parse_package(package)
    session = _get_session(team)

    lookup_url = "{url}/api/access/{owner}/{pkg}/".format(url=get_registry_url(team), owner=owner, pkg=pkg)
    response = session.get(lookup_url)

    data = response.json()
    users = data['users']

    print('\n'.join(users))

def access_add(package, user):
    """
    Add access
    """
    team, owner, pkg = parse_package(package)
    session = _get_session(team)

    session.put("%s/api/access/%s/%s/%s" % (get_registry_url(team), owner, pkg, user))
    print(u'Access added for %s' % user)

def access_remove(package, user):
    """
    Remove access
    """
    team, owner, pkg = parse_package(package)
    session = _get_session(team)

    session.delete("%s/api/access/%s/%s/%s" % (get_registry_url(team), owner, pkg, user))
    print(u'Access removed for %s' % user)

def delete(package):
    """
    Delete a package from the server.

    Irreversibly deletes the package along with its history, tags, versions, etc.
    """
    team, owner, pkg = parse_package(package)

    answer = input(
        "Are you sure you want to delete this package and its entire history? "
        "Type '%s' to confirm: " % package
    )

    if answer != package:
        print("Not deleting.")
        return 1

    session = _get_session(team)

    session.delete("%s/api/package/%s/%s/" % (get_registry_url(team), owner, pkg))
    print("Deleted.")

def search(query, team=None):
    """
    Search for packages
    """
    if team is None:
        team = _find_logged_in_team()

    if team is not None:
        session = _get_session(team)
        response = session.get("%s/api/search/" % get_registry_url(team), params=dict(q=query))
        print("* Packages in team %s" % team)
        packages = response.json()['packages']
        for pkg in packages:
            print(("%s:" % team) + ("%(owner)s/%(name)s" % pkg))
        if len(packages) == 0:
            print("(No results)")
        print("* Packages in public cloud")

    public_session = _get_session(None)
    response = public_session.get("%s/api/search/" % get_registry_url(None), params=dict(q=query))
    packages = response.json()['packages']
    for pkg in packages:
        print("%(owner)s/%(name)s" % pkg)
    if len(packages) == 0:
        print("(No results)")

def ls():                       # pylint:disable=C0103
    """
    List all installed Quilt data packages
    """
    for pkg_dir in PackageStore.find_store_dirs():
        print("%s" % pkg_dir)
        packages = PackageStore(pkg_dir).ls_packages()
        for package, tag, pkghash in sorted(packages):
            print("{0:30} {1:20} {2}".format(package, tag, pkghash))

def inspect(package):
    """
    Inspect package details
    """
    team, owner, pkg = parse_package(package)

    store, pkgroot = PackageStore.find_package(team, owner, pkg)
    if pkgroot is None:
        raise CommandException("Package {package} not found.".format(package=package))

    def _print_children(children, prefix, path):
        for idx, (name, child) in enumerate(children):
            if idx == len(children) - 1:
                new_prefix = u"└─"
                new_child_prefix = u"  "
            else:
                new_prefix = u"├─"
                new_child_prefix = u"│ "
            _print_node(child, prefix + new_prefix, prefix + new_child_prefix, name, path)

    def _print_node(node, prefix, child_prefix, name, path):
        name_prefix = u"─ "
        if isinstance(node, GroupNode):
            children = list(node.children.items())
            if children:
                name_prefix = u"┬ "
            print(prefix + name_prefix + name)
            _print_children(children, child_prefix, path + name)
        elif node.metadata['q_target'] == TargetType.PANDAS.value:
            df = store.load_dataframe(node.hashes)
            assert isinstance(df, pd.DataFrame)
            types = ", ".join("%r: %s" % (name, dtype) for name, dtype in df.dtypes.items())
            if len(types) > 64:
                types = types[:63] + "…"
            info = "shape %s, types %s" % (df.shape, types)
            print(prefix + name_prefix + name + ": " + info)
        else:
            print(prefix + name_prefix + name)

    print(store.package_path(team, owner, pkg))
    _print_children(children=pkgroot.children.items(), prefix='', path='')

def rm(package, force=False):
    """
    Remove a package (all instances) from the local store.
    """
    team, owner, pkg = parse_package(package)

    if not force:
        confirmed = input("Remove {0}? (y/n) ".format(package))
        if confirmed.lower() != 'y':
            return

    store = PackageStore()
    deleted = store.remove_package(team, owner, pkg)
    for obj in deleted:
        print("Removed: {0}".format(obj))

def list_users(team=None):
    # get team from disk if not specified
    if team is None:
        team = _find_logged_in_team()
    session = _get_session(team)
    url = get_registry_url(team)
    resp = session.get('%s/api/users/list' % url)
    return resp.json()

def _print_table(table, padding=2):
    cols_width = [max(len(word) for word in col) for col in zip(*table)]
    for row in table:
        print((" " * padding).join(word.ljust(width) for word, width in zip(row, cols_width)))

def _cli_list_users(team=None):
    res = list_users(team)
    l = [['Name', 'Email', 'Active', 'Superuser']]
    for user in res.get('results'):
        name = user.get('username')
        email = user.get('email')
        active = user.get('is_active')
        su = user.get('is_superuser')
        l.append([name, email, str(active), str(su)])

    _print_table(l)

def list_users_detailed(team=None):
    # get team from disk if not specified
    if team is None:
        team = _find_logged_in_team()
    session = _get_session(team)
    url = get_registry_url(team)
    resp = session.get('%s/api/users/list_detailed' % url)
    return resp.json()

def create_user(username, email, team):
    _check_team_id(team)
    session = _get_session(team)
    url = get_registry_url(team)
    session.post('%s/api/users/create' % url,
            data=json.dumps({'username':username, 'email':email}))

def list_packages(username, team=None):
    # get team from disk if not specified
    if team is None:
        team = _find_logged_in_team()
    session = _get_session(team)
    url = get_registry_url(team)
    resp = session.get('%s/api/admin/package_list/%s' % (url, username))
    return resp.json()

def disable_user(username, team):
    _check_team_id(team)
    session = _get_session(team)
    url = get_registry_url(team)
    session.post('%s/api/users/disable' % url,
            data=json.dumps({'username':username}))

def enable_user(username, team):
    _check_team_id(team)
    session = _get_session(team)
    url = get_registry_url(team)
    session.post('%s/api/users/enable' % url,
            data=json.dumps({'username':username}))

def delete_user(username, team, force=False):
    _check_team_id(team)
    if not force:
        confirmed = input("Really delete user '{0}'? (y/n)".format(username))
        if confirmed.lower() != 'y':
            return

    session = _get_session(team)
    url = get_registry_url(team)
    session.post('%s/api/users/delete' % url, data=json.dumps({'username':username}))

def audit(user_or_package):
    parts = user_or_package.split('/')
    if len(parts) > 2 or not all(is_nodename(part) for part in parts):
        raise CommandException("Need either a user or a user/package")

    team = _find_logged_in_team()
    if not team:
        raise CommandException("Not logged in as a team user")

    session = _get_session(team)
    response = session.get(
        "{url}/api/audit/{user_or_package}/".format(
            url=get_registry_url(team),
            user_or_package=user_or_package
        )
    )

    return response.json().get('events')

def _cli_audit(user_or_package):
    events = audit(user_or_package)
    team = _find_logged_in_team()
    teamstr = '%s:' % team
    rows = [['Time', 'User', 'Package', 'Type']]
    for item in events:
        time = item.get('created')
        pretty_time = datetime.fromtimestamp(time).strftime(DTIMEF)
        user = item.get('user')
        pkg = '%s%s/%s' % (teamstr, item.get('package_owner'), item.get('package_name'))
        t = item.get('type')
        rows.append((pretty_time, user, pkg, t))

    _print_table(rows)

def reset_password(team, username):
    _check_team_id(team)
    session = _get_session(team)
    session.post(
        "{url}/api/users/reset_password".format(
            url=get_registry_url(team),
            ), data=json.dumps({'username':username})
    )

def _load(package, hash=None):
    info = parse_package_extended(package)
    # TODO: support tags & versions.
    if info.tag:
        raise CommandException("Loading packages by tag is not supported.")
    elif info.version:
        raise CommandException("Loading packages by version is not supported.")
    elif info.hash:
        raise CommandException("Use hash=HASH to specify package hash.")

    store, pkgroot = PackageStore.find_package(info.team,
                                               info.user,
                                               info.name,
                                               pkghash=hash)
    if pkgroot is None:
        raise CommandException("Package {package} not found.".format(package=package))
    node = _from_core_node(store, pkgroot)

    return node, pkgroot, info

def load(pkginfo, hash=None):
    """
    functional interface to "from quilt.data.USER import PKG"
    """
    node, pkgroot, info = _load(pkginfo, hash)
    for subnode_name in info.subpath:
        node = node[subnode_name]

    return node

def export(package, output_path='.', force=False, symlinks=False):
    """Export package file data.

    Exports specified node (or its children) to files.

    `symlinks`
    **Warning** This is an advanced feature, use at your own risk.
                You must have a thorough understanding of permissions,
                and it is your responsibility to ensure that the linked
                files are not modified.  If they do become modified, it
                may corrupt your package store, and you may lose data,
                or you may use or distribute incorrect data.
    If `symlinks` is `True`:
      * Nodes that point to binary data will be symlinked instead of copied.
      * All other specified nodes (columnar data) will be exported as normal.

    :param package: package/subpackage name, e.g., user/foo or user/foo/bar
    :param output_path: distination folder
    :param symlinks: Use at your own risk.  See full description above.
    :param force: if True, overwrite existing files
    """
    # TODO: (future) Support other tags/versions (via load(), probably)
    # TODO: (future) This would be *drastically* simplified by a 1:1 source-file to node-name correlation
    # TODO: (future) This would be significantly simplified if node objects with useful accessors existed
    #       (nodes.Node and subclasses are being phased out, per Kevin)

    if symlinks is True:
        from quilt import _DEV_MODE
        if not _DEV_MODE:
            response = input("Warning: Exporting using symlinks to the package store.\n"
                "\tThis is an advanced feature.\n"
                "\tThe package store must not be written to directly, or it may be corrupted.\n"
                "\tManaging permissions and usage are your responsibility.\n\n"
                "Are you sure you want to continue? (yes/No) ")
            if response != 'yes':
                raise CommandException("No action taken: 'yes' not given")
    ## Helpers
    # Perhaps better as Node.iteritems()
    def iteritems(node, base_path=None, recursive=False):
        if base_path is None:
            base_path = pathlib.PureWindowsPath()
        assert isinstance(node, nodes.GroupNode)

        for name, child_node in node._items():
            child_path = base_path / name
            yield child_path, child_node
            if recursive and isinstance(child_node, nodes.GroupNode):
                for subpath, subnode in iteritems(child_node, child_path, recursive):
                    yield subpath, subnode

    # Perhaps better as Node.export_path
    def get_export_path(node, node_path):
        # If filepath is not present, generate fake path based on node parentage.
        filepath = node._meta[SYSTEM_METADATA]['filepath']
        if filepath:
            dest = pathlib.PureWindowsPath(filepath)  # PureWindowsPath handles all win/lin/osx separators
        else:
            assert isinstance(node_path, pathlib.PureWindowsPath)
            assert not node_path.anchor
            print("Warning:  Missing export path in metadata.  Using node path: {}"
                  .format('/'.join(node_path.parts)))
            dest = node_path

        # When exporting dataframes, excel files are to be converted to csv.
        # check also occurs in export_node(), but done here prevents filename conflicts
        if node._target() == TargetType.PANDAS:
            if dest.suffix != '.csv':
                # avoid name collisions from files with same name but different source,
                # as we shift all to being csv for export.
                # foo.tsv -> foo_tsv.csv
                # foo.xls -> foo_xls.csv
                # ..etc.
                dest = dest.with_name(dest.stem + dest.suffix.replace('.', '_')).with_suffix('.csv')
        # if filepath isn't absolute
        if not dest.anchor:
            return pathlib.Path(*dest.parts)  # return a native path

        # filepath is absolute, convert to relative.
        dest = pathlib.Path(*dest.parts[1:])  # Issue warning as native path, and return it
        print("Warning:  Converted export path to relative path: {}".format(str(dest)))
        return dest

    def iter_filename_map(node, base_path):
        """Yields (<node>, <export path>) pairs for given `node`.

        If `node` is a group node, yield pairs for children of `node`.
        Otherwise, yield the path to export to for that node.

        :returns: Iterator of (<node>, <export path>) pairs
        """
        # Handle singular node export
        if not isinstance(node, nodes.GroupNode):
            yield (node, get_export_path(node, base_path))
            return

        for node_path, found_node in iteritems(node, base_path=base_path, recursive=True):
            if not isinstance(found_node, nodes.GroupNode):
                yield (found_node, get_export_path(found_node, node_path))

    # perhaps better as Node.export()
    def export_node(node, dest, use_symlinks=False):
        if not dest.parent.exists():
            dest.parent.mkdir(parents=True, exist_ok=True)
        if node._target() == TargetType.FILE:
            if use_symlinks is True:
                fs_link(node(), dest)
            else:
                copyfile(node(), str(dest))
        elif node._target() == TargetType.PANDAS:
            df = node()
            # 100 decimal places of pi will allow you to draw a circle the size of the known
            # universe, and only vary by approximately the width of a proton.
            # ..so, hopefully 78 decimal places (256 bits) is ok for float precision in CSV exports.
            # If not, and someone complains, we can up it or add a parameter.
            df.to_csv(str(dest), index=False, float_format='%r')
        else:
            assert False

    def resolve_dirpath(dirpath):
        """Checks the dirpath and ensures it exists and is writable
        :returns: absolute, resolved dirpath
        """
        # ensure output path is writable.  I'd just check stat, but this is fully portable.
        try:
            dirpath.mkdir(exist_ok=True)  # could be '.'
            with tempfile.TemporaryFile(dir=str(dirpath), prefix="quilt-export-write-test-", suffix='.tmp'):
                pass
        except OSError as error:
            raise CommandException("Invalid export path: not writable: " + str(error))
        return output_path.resolve()    # this gets the true absolute path, but requires the path exists.

    def finalize(outpath, exports):
        """Finalize exports

        This performs the following tasks:
            * Ensure destination doesn't exist
            * Remove absolute anchor in dest, if any
            * Prefix destination with target dir

        :returns: (<source Path / dest Path pairs list>, <set of zero-byte files>)
        """
        # We return list instead of yielding, so that all prep logic is done before write is attempted.
        final_export_map = []

        for node, dest in exports:
            dest = pathlib.PureWindowsPath(dest)
            dest = outpath.joinpath(*dest.parts[1:]) if dest.anchor else outpath / dest

            if dest.parent != outpath and dest.parent.exists() and not force:
                raise CommandException("Invalid export path: subdir already exists: {!r}"
                                       .format(str(dest.parent)))
            if dest.exists() and not force:
                raise CommandException("Invalid export path: file already exists: {!r}".format(str(dest)))

            final_export_map.append((node, dest))
        return final_export_map

    def check_for_conflicts(export_list):
        """Checks for conflicting exports in the final export map of (src, dest) pairs

        Export conflicts can be introduced in various ways -- for example:
            * export-time mapping -- user maps two files to the same name
            * coded builds -- user creates two files with the same path
            * re-rooting absolute paths -- user entered absolute paths, which are re-rooted to the export dir
            * build-time duplication -- user enters the same file path twice under different nodes

        This checks for these conflicts and raises an error if they have occurred.

        :raises: CommandException
        """
        results = {}
        conflicts = set()

        # Export conflicts..
        for src, dest in export_list:
            if dest in conflicts:
                continue    # already a known conflict
            if dest not in results:
                results[dest] = src
                continue    # not a conflict..
            if src._target() == TargetType.FILE and src() == results[dest]():
                continue    # not a conflict (same src filename, same dest)..
            # ..add other conditions that prevent this from being a conflict here..

            # dest is a conflict.
            conflicts.add(dest)

        if conflicts:
            conflict_strings = (os.linesep + '\t').join(str(c) for c in conflicts)
            conflict_error = CommandException(
                "Invalid export: Identical filename(s) with conflicting contents cannot be exported:\n\t"
                + conflict_strings
                )
            conflict_error.file_conflicts = conflicts
            raise conflict_error

        # Check for filenames that conflict with folder names
        exports = set(results)
        dirs = set()
        for dest in results:
            dirs.update(dest.parents)
        file_dir_conflicts = exports & dirs

        if file_dir_conflicts:
            conflict_strings = (os.linesep + '\t').join(str(c) for c in file_dir_conflicts)
            conflict_error = CommandException(
                "Invalid Export: Filename(s) conflict with folder name(s):\n\t" + conflict_strings
                )
            conflict_error.dir_file_conflicts = file_dir_conflicts
            raise conflict_error
        # TODO: return abbreviated list of exports based on found non-conflicting duplicates

    ## Export Logic
    output_path = pathlib.Path(output_path)
    node, _, info = _load(package)

    if info.subpath:
        subpath = pathlib.PureWindowsPath(*info.subpath)
        for name in info.subpath:
            node = node._get(name)
    else:
        subpath = pathlib.PureWindowsPath()

    resolved_output = resolve_dirpath(output_path)  # resolve/create output path
    exports = iter_filename_map(node, subpath)      # Create src / dest map iterator
    exports = finalize(resolved_output, exports)    # Fix absolutes, check dest nonexistent, prefix dest dir
    check_for_conflicts(exports)                    # Prevent various potential dir/naming conflicts

    # Skip it if there's nothing to do
    if not exports:
        # Technically successful, but with nothing to do.
        # package may have no file nodes, or user may have filtered out all applicable targets.
        # -- should we consider it an error and raise?
        print("No files to export.")
        return

    # All prep done, let's export..
    try:
        fmt = "Exporting file {n_fmt} of {total_fmt} [{elapsed}]"
        sys.stdout.flush()   # flush prior text before making progress bar

        # bar_format is not respected unless both ncols and total are set.
        exports_bar = tqdm(exports, desc="Exporting: ", ncols=1, total=len(exports), bar_format=fmt)
        # tqdm is threaded, and its display may not specify the exact file currently being exported.
        with exports_bar:
            for node, dest in exports_bar:
                # Escape { and } in filenames for .format called by tqdm
                fname = str(dest.relative_to(resolved_output)).replace('{', "{{").replace('}', '}}')
                exports_bar.bar_format = fmt + ": " + fname
                exports_bar.update(0)
                export_node(node, dest, use_symlinks=symlinks)
    except OSError as error:
        commandex = CommandException("Unexpected error during export: " + str(error))
        commandex.original_error = error
        raise commandex
